Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * driver for Microchip PQI-based storage controllers
4 * Copyright (c) 2019-2023 Microchip Technology Inc. and its subsidiaries
5 * Copyright (c) 2016-2018 Microsemi Corporation
6 * Copyright (c) 2016 PMC-Sierra, Inc.
7 *
8 * Questions/Comments/Bugfixes to storagedev@microchip.com
9 *
10 */
11
12#include <linux/module.h>
13#include <linux/kernel.h>
14#include <linux/pci.h>
15#include <linux/delay.h>
16#include <linux/interrupt.h>
17#include <linux/sched.h>
18#include <linux/rtc.h>
19#include <linux/bcd.h>
20#include <linux/reboot.h>
21#include <linux/cciss_ioctl.h>
22#include <linux/crash_dump.h>
23#include <linux/string.h>
24#include <scsi/scsi_host.h>
25#include <scsi/scsi_cmnd.h>
26#include <scsi/scsi_device.h>
27#include <scsi/scsi_eh.h>
28#include <scsi/scsi_transport_sas.h>
29#include <linux/unaligned.h>
30#include "smartpqi.h"
31#include "smartpqi_sis.h"
32
33#if !defined(BUILD_TIMESTAMP)
34#define BUILD_TIMESTAMP
35#endif
36
37#define DRIVER_VERSION "2.1.36-026"
38#define DRIVER_MAJOR 2
39#define DRIVER_MINOR 1
40#define DRIVER_RELEASE 36
41#define DRIVER_REVISION 26
42
43#define DRIVER_NAME "Microchip SmartPQI Driver (v" \
44 DRIVER_VERSION BUILD_TIMESTAMP ")"
45#define DRIVER_NAME_SHORT "smartpqi"
46
47#define PQI_EXTRA_SGL_MEMORY (12 * sizeof(struct pqi_sg_descriptor))
48
49#define PQI_POST_RESET_DELAY_SECS 5
50#define PQI_POST_OFA_RESET_DELAY_UPON_TIMEOUT_SECS 10
51
52#define PQI_NO_COMPLETION ((void *)-1)
53
54MODULE_AUTHOR("Microchip");
55MODULE_DESCRIPTION("Driver for Microchip Smart Family Controller version "
56 DRIVER_VERSION);
57MODULE_VERSION(DRIVER_VERSION);
58MODULE_LICENSE("GPL");
59
60struct pqi_cmd_priv {
61 int this_residual;
62};
63
64static struct pqi_cmd_priv *pqi_cmd_priv(struct scsi_cmnd *cmd)
65{
66 return scsi_cmd_priv(cmd);
67}
68
69static void pqi_verify_structures(void);
70static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info,
71 enum pqi_ctrl_shutdown_reason ctrl_shutdown_reason);
72static void pqi_take_ctrl_devices_offline(struct pqi_ctrl_info *ctrl_info);
73static void pqi_ctrl_offline_worker(struct work_struct *work);
74static int pqi_scan_scsi_devices(struct pqi_ctrl_info *ctrl_info);
75static void pqi_scan_start(struct Scsi_Host *shost);
76static void pqi_start_io(struct pqi_ctrl_info *ctrl_info,
77 struct pqi_queue_group *queue_group, enum pqi_io_path path,
78 struct pqi_io_request *io_request);
79static int pqi_submit_raid_request_synchronous(struct pqi_ctrl_info *ctrl_info,
80 struct pqi_iu_header *request, unsigned int flags,
81 struct pqi_raid_error_info *error_info);
82static int pqi_aio_submit_io(struct pqi_ctrl_info *ctrl_info,
83 struct scsi_cmnd *scmd, u32 aio_handle, u8 *cdb,
84 unsigned int cdb_length, struct pqi_queue_group *queue_group,
85 struct pqi_encryption_info *encryption_info, bool raid_bypass, bool io_high_prio);
86static int pqi_aio_submit_r1_write_io(struct pqi_ctrl_info *ctrl_info,
87 struct scsi_cmnd *scmd, struct pqi_queue_group *queue_group,
88 struct pqi_encryption_info *encryption_info, struct pqi_scsi_dev *device,
89 struct pqi_scsi_dev_raid_map_data *rmd);
90static int pqi_aio_submit_r56_write_io(struct pqi_ctrl_info *ctrl_info,
91 struct scsi_cmnd *scmd, struct pqi_queue_group *queue_group,
92 struct pqi_encryption_info *encryption_info, struct pqi_scsi_dev *device,
93 struct pqi_scsi_dev_raid_map_data *rmd);
94static void pqi_ofa_ctrl_quiesce(struct pqi_ctrl_info *ctrl_info);
95static void pqi_ofa_ctrl_unquiesce(struct pqi_ctrl_info *ctrl_info);
96static int pqi_ofa_ctrl_restart(struct pqi_ctrl_info *ctrl_info, unsigned int delay_secs);
97static void pqi_host_setup_buffer(struct pqi_ctrl_info *ctrl_info, struct pqi_host_memory_descriptor *host_memory_descriptor, u32 total_size, u32 min_size);
98static void pqi_host_free_buffer(struct pqi_ctrl_info *ctrl_info, struct pqi_host_memory_descriptor *host_memory_descriptor);
99static int pqi_host_memory_update(struct pqi_ctrl_info *ctrl_info, struct pqi_host_memory_descriptor *host_memory_descriptor, u16 function_code);
100static int pqi_device_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info,
101 struct pqi_scsi_dev *device, u8 lun, unsigned long timeout_msecs);
102static void pqi_fail_all_outstanding_requests(struct pqi_ctrl_info *ctrl_info);
103static void pqi_tmf_worker(struct work_struct *work);
104
105/* for flags argument to pqi_submit_raid_request_synchronous() */
106#define PQI_SYNC_FLAGS_INTERRUPTABLE 0x1
107
108static struct scsi_transport_template *pqi_sas_transport_template;
109
110static atomic_t pqi_controller_count = ATOMIC_INIT(0);
111
112enum pqi_lockup_action {
113 NONE,
114 REBOOT,
115 PANIC
116};
117
118static enum pqi_lockup_action pqi_lockup_action = NONE;
119
120static struct {
121 enum pqi_lockup_action action;
122 char *name;
123} pqi_lockup_actions[] = {
124 {
125 .action = NONE,
126 .name = "none",
127 },
128 {
129 .action = REBOOT,
130 .name = "reboot",
131 },
132 {
133 .action = PANIC,
134 .name = "panic",
135 },
136};
137
138static unsigned int pqi_supported_event_types[] = {
139 PQI_EVENT_TYPE_HOTPLUG,
140 PQI_EVENT_TYPE_HARDWARE,
141 PQI_EVENT_TYPE_PHYSICAL_DEVICE,
142 PQI_EVENT_TYPE_LOGICAL_DEVICE,
143 PQI_EVENT_TYPE_OFA,
144 PQI_EVENT_TYPE_AIO_STATE_CHANGE,
145 PQI_EVENT_TYPE_AIO_CONFIG_CHANGE,
146};
147
148static int pqi_disable_device_id_wildcards;
149module_param_named(disable_device_id_wildcards,
150 pqi_disable_device_id_wildcards, int, 0644);
151MODULE_PARM_DESC(disable_device_id_wildcards,
152 "Disable device ID wildcards.");
153
154static int pqi_disable_heartbeat;
155module_param_named(disable_heartbeat,
156 pqi_disable_heartbeat, int, 0644);
157MODULE_PARM_DESC(disable_heartbeat,
158 "Disable heartbeat.");
159
160static int pqi_disable_ctrl_shutdown;
161module_param_named(disable_ctrl_shutdown,
162 pqi_disable_ctrl_shutdown, int, 0644);
163MODULE_PARM_DESC(disable_ctrl_shutdown,
164 "Disable controller shutdown when controller locked up.");
165
166static char *pqi_lockup_action_param;
167module_param_named(lockup_action,
168 pqi_lockup_action_param, charp, 0644);
169MODULE_PARM_DESC(lockup_action, "Action to take when controller locked up.\n"
170 "\t\tSupported: none, reboot, panic\n"
171 "\t\tDefault: none");
172
173static int pqi_expose_ld_first;
174module_param_named(expose_ld_first,
175 pqi_expose_ld_first, int, 0644);
176MODULE_PARM_DESC(expose_ld_first, "Expose logical drives before physical drives.");
177
178static int pqi_hide_vsep;
179module_param_named(hide_vsep,
180 pqi_hide_vsep, int, 0644);
181MODULE_PARM_DESC(hide_vsep, "Hide the virtual SEP for direct attached drives.");
182
183static int pqi_disable_managed_interrupts;
184module_param_named(disable_managed_interrupts,
185 pqi_disable_managed_interrupts, int, 0644);
186MODULE_PARM_DESC(disable_managed_interrupts,
187 "Disable the kernel automatically assigning SMP affinity to IRQs.");
188
189static unsigned int pqi_ctrl_ready_timeout_secs;
190module_param_named(ctrl_ready_timeout,
191 pqi_ctrl_ready_timeout_secs, uint, 0644);
192MODULE_PARM_DESC(ctrl_ready_timeout,
193 "Timeout in seconds for driver to wait for controller ready.");
194
195static char *raid_levels[] = {
196 "RAID-0",
197 "RAID-4",
198 "RAID-1(1+0)",
199 "RAID-5",
200 "RAID-5+1",
201 "RAID-6",
202 "RAID-1(Triple)",
203};
204
205static char *pqi_raid_level_to_string(u8 raid_level)
206{
207 if (raid_level < ARRAY_SIZE(raid_levels))
208 return raid_levels[raid_level];
209
210 return "RAID UNKNOWN";
211}
212
213#define SA_RAID_0 0
214#define SA_RAID_4 1
215#define SA_RAID_1 2 /* also used for RAID 10 */
216#define SA_RAID_5 3 /* also used for RAID 50 */
217#define SA_RAID_51 4
218#define SA_RAID_6 5 /* also used for RAID 60 */
219#define SA_RAID_TRIPLE 6 /* also used for RAID 1+0 Triple */
220#define SA_RAID_MAX SA_RAID_TRIPLE
221#define SA_RAID_UNKNOWN 0xff
222
223static inline void pqi_scsi_done(struct scsi_cmnd *scmd)
224{
225 pqi_prep_for_scsi_done(scmd);
226 scsi_done(scmd);
227}
228
229static inline void pqi_disable_write_same(struct scsi_device *sdev)
230{
231 sdev->no_write_same = 1;
232}
233
234static inline bool pqi_scsi3addr_equal(u8 *scsi3addr1, u8 *scsi3addr2)
235{
236 return memcmp(scsi3addr1, scsi3addr2, 8) == 0;
237}
238
239static inline bool pqi_is_logical_device(struct pqi_scsi_dev *device)
240{
241 return !device->is_physical_device;
242}
243
244static inline bool pqi_is_external_raid_addr(u8 *scsi3addr)
245{
246 return scsi3addr[2] != 0;
247}
248
249static inline bool pqi_ctrl_offline(struct pqi_ctrl_info *ctrl_info)
250{
251 return !ctrl_info->controller_online;
252}
253
254static inline void pqi_check_ctrl_health(struct pqi_ctrl_info *ctrl_info)
255{
256 if (ctrl_info->controller_online)
257 if (!sis_is_firmware_running(ctrl_info))
258 pqi_take_ctrl_offline(ctrl_info, PQI_FIRMWARE_KERNEL_NOT_UP);
259}
260
261static inline bool pqi_is_hba_lunid(u8 *scsi3addr)
262{
263 return pqi_scsi3addr_equal(scsi3addr, RAID_CTLR_LUNID);
264}
265
266#define PQI_DRIVER_SCRATCH_PQI_MODE 0x1
267#define PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED 0x2
268
269static inline enum pqi_ctrl_mode pqi_get_ctrl_mode(struct pqi_ctrl_info *ctrl_info)
270{
271 return sis_read_driver_scratch(ctrl_info) & PQI_DRIVER_SCRATCH_PQI_MODE ? PQI_MODE : SIS_MODE;
272}
273
274static inline void pqi_save_ctrl_mode(struct pqi_ctrl_info *ctrl_info,
275 enum pqi_ctrl_mode mode)
276{
277 u32 driver_scratch;
278
279 driver_scratch = sis_read_driver_scratch(ctrl_info);
280
281 if (mode == PQI_MODE)
282 driver_scratch |= PQI_DRIVER_SCRATCH_PQI_MODE;
283 else
284 driver_scratch &= ~PQI_DRIVER_SCRATCH_PQI_MODE;
285
286 sis_write_driver_scratch(ctrl_info, driver_scratch);
287}
288
289static inline bool pqi_is_fw_triage_supported(struct pqi_ctrl_info *ctrl_info)
290{
291 return (sis_read_driver_scratch(ctrl_info) & PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED) != 0;
292}
293
294static inline void pqi_save_fw_triage_setting(struct pqi_ctrl_info *ctrl_info, bool is_supported)
295{
296 u32 driver_scratch;
297
298 driver_scratch = sis_read_driver_scratch(ctrl_info);
299
300 if (is_supported)
301 driver_scratch |= PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED;
302 else
303 driver_scratch &= ~PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED;
304
305 sis_write_driver_scratch(ctrl_info, driver_scratch);
306}
307
308static inline void pqi_ctrl_block_scan(struct pqi_ctrl_info *ctrl_info)
309{
310 ctrl_info->scan_blocked = true;
311 mutex_lock(&ctrl_info->scan_mutex);
312}
313
314static inline void pqi_ctrl_unblock_scan(struct pqi_ctrl_info *ctrl_info)
315{
316 ctrl_info->scan_blocked = false;
317 mutex_unlock(&ctrl_info->scan_mutex);
318}
319
320static inline bool pqi_ctrl_scan_blocked(struct pqi_ctrl_info *ctrl_info)
321{
322 return ctrl_info->scan_blocked;
323}
324
325static inline void pqi_ctrl_block_device_reset(struct pqi_ctrl_info *ctrl_info)
326{
327 mutex_lock(&ctrl_info->lun_reset_mutex);
328}
329
330static inline void pqi_ctrl_unblock_device_reset(struct pqi_ctrl_info *ctrl_info)
331{
332 mutex_unlock(&ctrl_info->lun_reset_mutex);
333}
334
335static inline void pqi_scsi_block_requests(struct pqi_ctrl_info *ctrl_info)
336{
337 struct Scsi_Host *shost;
338 unsigned int num_loops;
339 int msecs_sleep;
340
341 shost = ctrl_info->scsi_host;
342
343 scsi_block_requests(shost);
344
345 num_loops = 0;
346 msecs_sleep = 20;
347 while (scsi_host_busy(shost)) {
348 num_loops++;
349 if (num_loops == 10)
350 msecs_sleep = 500;
351 msleep(msecs_sleep);
352 }
353}
354
355static inline void pqi_scsi_unblock_requests(struct pqi_ctrl_info *ctrl_info)
356{
357 scsi_unblock_requests(ctrl_info->scsi_host);
358}
359
360static inline void pqi_ctrl_busy(struct pqi_ctrl_info *ctrl_info)
361{
362 atomic_inc(&ctrl_info->num_busy_threads);
363}
364
365static inline void pqi_ctrl_unbusy(struct pqi_ctrl_info *ctrl_info)
366{
367 atomic_dec(&ctrl_info->num_busy_threads);
368}
369
370static inline bool pqi_ctrl_blocked(struct pqi_ctrl_info *ctrl_info)
371{
372 return ctrl_info->block_requests;
373}
374
375static inline void pqi_ctrl_block_requests(struct pqi_ctrl_info *ctrl_info)
376{
377 ctrl_info->block_requests = true;
378}
379
380static inline void pqi_ctrl_unblock_requests(struct pqi_ctrl_info *ctrl_info)
381{
382 ctrl_info->block_requests = false;
383 wake_up_all(&ctrl_info->block_requests_wait);
384}
385
386static void pqi_wait_if_ctrl_blocked(struct pqi_ctrl_info *ctrl_info)
387{
388 if (!pqi_ctrl_blocked(ctrl_info))
389 return;
390
391 atomic_inc(&ctrl_info->num_blocked_threads);
392 wait_event(ctrl_info->block_requests_wait,
393 !pqi_ctrl_blocked(ctrl_info));
394 atomic_dec(&ctrl_info->num_blocked_threads);
395}
396
397#define PQI_QUIESCE_WARNING_TIMEOUT_SECS 10
398
399static inline void pqi_ctrl_wait_until_quiesced(struct pqi_ctrl_info *ctrl_info)
400{
401 unsigned long start_jiffies;
402 unsigned long warning_timeout;
403 bool displayed_warning;
404
405 displayed_warning = false;
406 start_jiffies = jiffies;
407 warning_timeout = (PQI_QUIESCE_WARNING_TIMEOUT_SECS * HZ) + start_jiffies;
408
409 while (atomic_read(&ctrl_info->num_busy_threads) >
410 atomic_read(&ctrl_info->num_blocked_threads)) {
411 if (time_after(jiffies, warning_timeout)) {
412 dev_warn(&ctrl_info->pci_dev->dev,
413 "waiting %u seconds for driver activity to quiesce\n",
414 jiffies_to_msecs(jiffies - start_jiffies) / 1000);
415 displayed_warning = true;
416 warning_timeout = (PQI_QUIESCE_WARNING_TIMEOUT_SECS * HZ) + jiffies;
417 }
418 usleep_range(1000, 2000);
419 }
420
421 if (displayed_warning)
422 dev_warn(&ctrl_info->pci_dev->dev,
423 "driver activity quiesced after waiting for %u seconds\n",
424 jiffies_to_msecs(jiffies - start_jiffies) / 1000);
425}
426
427static inline bool pqi_device_offline(struct pqi_scsi_dev *device)
428{
429 return device->device_offline;
430}
431
432static inline void pqi_ctrl_ofa_start(struct pqi_ctrl_info *ctrl_info)
433{
434 mutex_lock(&ctrl_info->ofa_mutex);
435}
436
437static inline void pqi_ctrl_ofa_done(struct pqi_ctrl_info *ctrl_info)
438{
439 mutex_unlock(&ctrl_info->ofa_mutex);
440}
441
442static inline void pqi_wait_until_ofa_finished(struct pqi_ctrl_info *ctrl_info)
443{
444 mutex_lock(&ctrl_info->ofa_mutex);
445 mutex_unlock(&ctrl_info->ofa_mutex);
446}
447
448static inline bool pqi_ofa_in_progress(struct pqi_ctrl_info *ctrl_info)
449{
450 return mutex_is_locked(&ctrl_info->ofa_mutex);
451}
452
453static inline void pqi_device_remove_start(struct pqi_scsi_dev *device)
454{
455 device->in_remove = true;
456}
457
458static inline bool pqi_device_in_remove(struct pqi_scsi_dev *device)
459{
460 return device->in_remove;
461}
462
463static inline void pqi_device_reset_start(struct pqi_scsi_dev *device, u8 lun)
464{
465 device->in_reset[lun] = true;
466}
467
468static inline void pqi_device_reset_done(struct pqi_scsi_dev *device, u8 lun)
469{
470 device->in_reset[lun] = false;
471}
472
473static inline bool pqi_device_in_reset(struct pqi_scsi_dev *device, u8 lun)
474{
475 return device->in_reset[lun];
476}
477
478static inline int pqi_event_type_to_event_index(unsigned int event_type)
479{
480 int index;
481
482 for (index = 0; index < ARRAY_SIZE(pqi_supported_event_types); index++)
483 if (event_type == pqi_supported_event_types[index])
484 return index;
485
486 return -1;
487}
488
489static inline bool pqi_is_supported_event(unsigned int event_type)
490{
491 return pqi_event_type_to_event_index(event_type) != -1;
492}
493
494static inline void pqi_schedule_rescan_worker_with_delay(struct pqi_ctrl_info *ctrl_info,
495 unsigned long delay)
496{
497 if (pqi_ctrl_offline(ctrl_info))
498 return;
499
500 schedule_delayed_work(&ctrl_info->rescan_work, delay);
501}
502
503static inline void pqi_schedule_rescan_worker(struct pqi_ctrl_info *ctrl_info)
504{
505 pqi_schedule_rescan_worker_with_delay(ctrl_info, 0);
506}
507
508#define PQI_RESCAN_WORK_DELAY (10 * HZ)
509
510static inline void pqi_schedule_rescan_worker_delayed(struct pqi_ctrl_info *ctrl_info)
511{
512 pqi_schedule_rescan_worker_with_delay(ctrl_info, PQI_RESCAN_WORK_DELAY);
513}
514
515static inline void pqi_cancel_rescan_worker(struct pqi_ctrl_info *ctrl_info)
516{
517 cancel_delayed_work_sync(&ctrl_info->rescan_work);
518}
519
520static inline u32 pqi_read_heartbeat_counter(struct pqi_ctrl_info *ctrl_info)
521{
522 if (!ctrl_info->heartbeat_counter)
523 return 0;
524
525 return readl(ctrl_info->heartbeat_counter);
526}
527
528static inline u8 pqi_read_soft_reset_status(struct pqi_ctrl_info *ctrl_info)
529{
530 return readb(ctrl_info->soft_reset_status);
531}
532
533static inline void pqi_clear_soft_reset_status(struct pqi_ctrl_info *ctrl_info)
534{
535 u8 status;
536
537 status = pqi_read_soft_reset_status(ctrl_info);
538 status &= ~PQI_SOFT_RESET_ABORT;
539 writeb(status, ctrl_info->soft_reset_status);
540}
541
542static inline bool pqi_is_io_high_priority(struct pqi_scsi_dev *device, struct scsi_cmnd *scmd)
543{
544 bool io_high_prio;
545 int priority_class;
546
547 io_high_prio = false;
548
549 if (device->ncq_prio_enable) {
550 priority_class =
551 IOPRIO_PRIO_CLASS(req_get_ioprio(scsi_cmd_to_rq(scmd)));
552 if (priority_class == IOPRIO_CLASS_RT) {
553 /* Set NCQ priority for read/write commands. */
554 switch (scmd->cmnd[0]) {
555 case WRITE_16:
556 case READ_16:
557 case WRITE_12:
558 case READ_12:
559 case WRITE_10:
560 case READ_10:
561 case WRITE_6:
562 case READ_6:
563 io_high_prio = true;
564 break;
565 }
566 }
567 }
568
569 return io_high_prio;
570}
571
572static int pqi_map_single(struct pci_dev *pci_dev,
573 struct pqi_sg_descriptor *sg_descriptor, void *buffer,
574 size_t buffer_length, enum dma_data_direction data_direction)
575{
576 dma_addr_t bus_address;
577
578 if (!buffer || buffer_length == 0 || data_direction == DMA_NONE)
579 return 0;
580
581 bus_address = dma_map_single(&pci_dev->dev, buffer, buffer_length,
582 data_direction);
583 if (dma_mapping_error(&pci_dev->dev, bus_address))
584 return -ENOMEM;
585
586 put_unaligned_le64((u64)bus_address, &sg_descriptor->address);
587 put_unaligned_le32(buffer_length, &sg_descriptor->length);
588 put_unaligned_le32(CISS_SG_LAST, &sg_descriptor->flags);
589
590 return 0;
591}
592
593static void pqi_pci_unmap(struct pci_dev *pci_dev,
594 struct pqi_sg_descriptor *descriptors, int num_descriptors,
595 enum dma_data_direction data_direction)
596{
597 int i;
598
599 if (data_direction == DMA_NONE)
600 return;
601
602 for (i = 0; i < num_descriptors; i++)
603 dma_unmap_single(&pci_dev->dev,
604 (dma_addr_t)get_unaligned_le64(&descriptors[i].address),
605 get_unaligned_le32(&descriptors[i].length),
606 data_direction);
607}
608
609static int pqi_build_raid_path_request(struct pqi_ctrl_info *ctrl_info,
610 struct pqi_raid_path_request *request, u8 cmd,
611 u8 *scsi3addr, void *buffer, size_t buffer_length,
612 u16 vpd_page, enum dma_data_direction *dir)
613{
614 u8 *cdb;
615 size_t cdb_length = buffer_length;
616
617 memset(request, 0, sizeof(*request));
618
619 request->header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO;
620 put_unaligned_le16(offsetof(struct pqi_raid_path_request,
621 sg_descriptors[1]) - PQI_REQUEST_HEADER_LENGTH,
622 &request->header.iu_length);
623 put_unaligned_le32(buffer_length, &request->buffer_length);
624 memcpy(request->lun_number, scsi3addr, sizeof(request->lun_number));
625 request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
626 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_0;
627
628 cdb = request->cdb;
629
630 switch (cmd) {
631 case INQUIRY:
632 request->data_direction = SOP_READ_FLAG;
633 cdb[0] = INQUIRY;
634 if (vpd_page & VPD_PAGE) {
635 cdb[1] = 0x1;
636 cdb[2] = (u8)vpd_page;
637 }
638 cdb[4] = (u8)cdb_length;
639 break;
640 case CISS_REPORT_LOG:
641 case CISS_REPORT_PHYS:
642 request->data_direction = SOP_READ_FLAG;
643 cdb[0] = cmd;
644 if (cmd == CISS_REPORT_PHYS) {
645 if (ctrl_info->rpl_extended_format_4_5_supported)
646 cdb[1] = CISS_REPORT_PHYS_FLAG_EXTENDED_FORMAT_4;
647 else
648 cdb[1] = CISS_REPORT_PHYS_FLAG_EXTENDED_FORMAT_2;
649 } else {
650 cdb[1] = ctrl_info->ciss_report_log_flags;
651 }
652 put_unaligned_be32(cdb_length, &cdb[6]);
653 break;
654 case CISS_GET_RAID_MAP:
655 request->data_direction = SOP_READ_FLAG;
656 cdb[0] = CISS_READ;
657 cdb[1] = CISS_GET_RAID_MAP;
658 put_unaligned_be32(cdb_length, &cdb[6]);
659 break;
660 case SA_FLUSH_CACHE:
661 request->header.driver_flags = PQI_DRIVER_NONBLOCKABLE_REQUEST;
662 request->data_direction = SOP_WRITE_FLAG;
663 cdb[0] = BMIC_WRITE;
664 cdb[6] = BMIC_FLUSH_CACHE;
665 put_unaligned_be16(cdb_length, &cdb[7]);
666 break;
667 case BMIC_SENSE_DIAG_OPTIONS:
668 cdb_length = 0;
669 fallthrough;
670 case BMIC_IDENTIFY_CONTROLLER:
671 case BMIC_IDENTIFY_PHYSICAL_DEVICE:
672 case BMIC_SENSE_SUBSYSTEM_INFORMATION:
673 case BMIC_SENSE_FEATURE:
674 request->data_direction = SOP_READ_FLAG;
675 cdb[0] = BMIC_READ;
676 cdb[6] = cmd;
677 put_unaligned_be16(cdb_length, &cdb[7]);
678 break;
679 case BMIC_SET_DIAG_OPTIONS:
680 cdb_length = 0;
681 fallthrough;
682 case BMIC_WRITE_HOST_WELLNESS:
683 request->data_direction = SOP_WRITE_FLAG;
684 cdb[0] = BMIC_WRITE;
685 cdb[6] = cmd;
686 put_unaligned_be16(cdb_length, &cdb[7]);
687 break;
688 case BMIC_CSMI_PASSTHRU:
689 request->data_direction = SOP_BIDIRECTIONAL;
690 cdb[0] = BMIC_WRITE;
691 cdb[5] = CSMI_CC_SAS_SMP_PASSTHRU;
692 cdb[6] = cmd;
693 put_unaligned_be16(cdb_length, &cdb[7]);
694 break;
695 default:
696 dev_err(&ctrl_info->pci_dev->dev, "unknown command 0x%c\n", cmd);
697 break;
698 }
699
700 switch (request->data_direction) {
701 case SOP_READ_FLAG:
702 *dir = DMA_FROM_DEVICE;
703 break;
704 case SOP_WRITE_FLAG:
705 *dir = DMA_TO_DEVICE;
706 break;
707 case SOP_NO_DIRECTION_FLAG:
708 *dir = DMA_NONE;
709 break;
710 default:
711 *dir = DMA_BIDIRECTIONAL;
712 break;
713 }
714
715 return pqi_map_single(ctrl_info->pci_dev, &request->sg_descriptors[0],
716 buffer, buffer_length, *dir);
717}
718
719static inline void pqi_reinit_io_request(struct pqi_io_request *io_request)
720{
721 io_request->scmd = NULL;
722 io_request->status = 0;
723 io_request->error_info = NULL;
724 io_request->raid_bypass = false;
725}
726
727static inline struct pqi_io_request *pqi_alloc_io_request(struct pqi_ctrl_info *ctrl_info, struct scsi_cmnd *scmd)
728{
729 struct pqi_io_request *io_request;
730 u16 i;
731
732 if (scmd) { /* SML I/O request */
733 u32 blk_tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd));
734
735 i = blk_mq_unique_tag_to_tag(blk_tag);
736 io_request = &ctrl_info->io_request_pool[i];
737 if (atomic_inc_return(&io_request->refcount) > 1) {
738 atomic_dec(&io_request->refcount);
739 return NULL;
740 }
741 } else { /* IOCTL or driver internal request */
742 /*
743 * benignly racy - may have to wait for an open slot.
744 * command slot range is scsi_ml_can_queue -
745 * [scsi_ml_can_queue + (PQI_RESERVED_IO_SLOTS - 1)]
746 */
747 i = 0;
748 while (1) {
749 io_request = &ctrl_info->io_request_pool[ctrl_info->scsi_ml_can_queue + i];
750 if (atomic_inc_return(&io_request->refcount) == 1)
751 break;
752 atomic_dec(&io_request->refcount);
753 i = (i + 1) % PQI_RESERVED_IO_SLOTS;
754 }
755 }
756
757 if (io_request)
758 pqi_reinit_io_request(io_request);
759
760 return io_request;
761}
762
763static void pqi_free_io_request(struct pqi_io_request *io_request)
764{
765 atomic_dec(&io_request->refcount);
766}
767
768static int pqi_send_scsi_raid_request(struct pqi_ctrl_info *ctrl_info, u8 cmd,
769 u8 *scsi3addr, void *buffer, size_t buffer_length, u16 vpd_page,
770 struct pqi_raid_error_info *error_info)
771{
772 int rc;
773 struct pqi_raid_path_request request;
774 enum dma_data_direction dir;
775
776 rc = pqi_build_raid_path_request(ctrl_info, &request, cmd, scsi3addr,
777 buffer, buffer_length, vpd_page, &dir);
778 if (rc)
779 return rc;
780
781 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, error_info);
782
783 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir);
784
785 return rc;
786}
787
788/* helper functions for pqi_send_scsi_raid_request */
789
790static inline int pqi_send_ctrl_raid_request(struct pqi_ctrl_info *ctrl_info,
791 u8 cmd, void *buffer, size_t buffer_length)
792{
793 return pqi_send_scsi_raid_request(ctrl_info, cmd, RAID_CTLR_LUNID,
794 buffer, buffer_length, 0, NULL);
795}
796
797static inline int pqi_send_ctrl_raid_with_error(struct pqi_ctrl_info *ctrl_info,
798 u8 cmd, void *buffer, size_t buffer_length,
799 struct pqi_raid_error_info *error_info)
800{
801 return pqi_send_scsi_raid_request(ctrl_info, cmd, RAID_CTLR_LUNID,
802 buffer, buffer_length, 0, error_info);
803}
804
805static inline int pqi_identify_controller(struct pqi_ctrl_info *ctrl_info,
806 struct bmic_identify_controller *buffer)
807{
808 return pqi_send_ctrl_raid_request(ctrl_info, BMIC_IDENTIFY_CONTROLLER,
809 buffer, sizeof(*buffer));
810}
811
812static inline int pqi_sense_subsystem_info(struct pqi_ctrl_info *ctrl_info,
813 struct bmic_sense_subsystem_info *sense_info)
814{
815 return pqi_send_ctrl_raid_request(ctrl_info,
816 BMIC_SENSE_SUBSYSTEM_INFORMATION, sense_info,
817 sizeof(*sense_info));
818}
819
820static inline int pqi_scsi_inquiry(struct pqi_ctrl_info *ctrl_info,
821 u8 *scsi3addr, u16 vpd_page, void *buffer, size_t buffer_length)
822{
823 return pqi_send_scsi_raid_request(ctrl_info, INQUIRY, scsi3addr,
824 buffer, buffer_length, vpd_page, NULL);
825}
826
827static int pqi_identify_physical_device(struct pqi_ctrl_info *ctrl_info,
828 struct pqi_scsi_dev *device,
829 struct bmic_identify_physical_device *buffer, size_t buffer_length)
830{
831 int rc;
832 enum dma_data_direction dir;
833 u16 bmic_device_index;
834 struct pqi_raid_path_request request;
835
836 rc = pqi_build_raid_path_request(ctrl_info, &request,
837 BMIC_IDENTIFY_PHYSICAL_DEVICE, RAID_CTLR_LUNID, buffer,
838 buffer_length, 0, &dir);
839 if (rc)
840 return rc;
841
842 bmic_device_index = CISS_GET_DRIVE_NUMBER(device->scsi3addr);
843 request.cdb[2] = (u8)bmic_device_index;
844 request.cdb[9] = (u8)(bmic_device_index >> 8);
845
846 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);
847
848 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir);
849
850 return rc;
851}
852
853static inline u32 pqi_aio_limit_to_bytes(__le16 *limit)
854{
855 u32 bytes;
856
857 bytes = get_unaligned_le16(limit);
858 if (bytes == 0)
859 bytes = ~0;
860 else
861 bytes *= 1024;
862
863 return bytes;
864}
865
866#pragma pack(1)
867
868struct bmic_sense_feature_buffer {
869 struct bmic_sense_feature_buffer_header header;
870 struct bmic_sense_feature_io_page_aio_subpage aio_subpage;
871};
872
873#pragma pack()
874
875#define MINIMUM_AIO_SUBPAGE_BUFFER_LENGTH \
876 offsetofend(struct bmic_sense_feature_buffer, \
877 aio_subpage.max_write_raid_1_10_3drive)
878
879#define MINIMUM_AIO_SUBPAGE_LENGTH \
880 (offsetofend(struct bmic_sense_feature_io_page_aio_subpage, \
881 max_write_raid_1_10_3drive) - \
882 sizeof_field(struct bmic_sense_feature_io_page_aio_subpage, header))
883
884static int pqi_get_advanced_raid_bypass_config(struct pqi_ctrl_info *ctrl_info)
885{
886 int rc;
887 enum dma_data_direction dir;
888 struct pqi_raid_path_request request;
889 struct bmic_sense_feature_buffer *buffer;
890
891 buffer = kmalloc_obj(*buffer);
892 if (!buffer)
893 return -ENOMEM;
894
895 rc = pqi_build_raid_path_request(ctrl_info, &request, BMIC_SENSE_FEATURE, RAID_CTLR_LUNID,
896 buffer, sizeof(*buffer), 0, &dir);
897 if (rc)
898 goto error;
899
900 request.cdb[2] = BMIC_SENSE_FEATURE_IO_PAGE;
901 request.cdb[3] = BMIC_SENSE_FEATURE_IO_PAGE_AIO_SUBPAGE;
902
903 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);
904
905 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir);
906
907 if (rc)
908 goto error;
909
910 if (buffer->header.page_code != BMIC_SENSE_FEATURE_IO_PAGE ||
911 buffer->header.subpage_code !=
912 BMIC_SENSE_FEATURE_IO_PAGE_AIO_SUBPAGE ||
913 get_unaligned_le16(&buffer->header.buffer_length) <
914 MINIMUM_AIO_SUBPAGE_BUFFER_LENGTH ||
915 buffer->aio_subpage.header.page_code !=
916 BMIC_SENSE_FEATURE_IO_PAGE ||
917 buffer->aio_subpage.header.subpage_code !=
918 BMIC_SENSE_FEATURE_IO_PAGE_AIO_SUBPAGE ||
919 get_unaligned_le16(&buffer->aio_subpage.header.page_length) <
920 MINIMUM_AIO_SUBPAGE_LENGTH) {
921 goto error;
922 }
923
924 ctrl_info->max_transfer_encrypted_sas_sata =
925 pqi_aio_limit_to_bytes(
926 &buffer->aio_subpage.max_transfer_encrypted_sas_sata);
927
928 ctrl_info->max_transfer_encrypted_nvme =
929 pqi_aio_limit_to_bytes(
930 &buffer->aio_subpage.max_transfer_encrypted_nvme);
931
932 ctrl_info->max_write_raid_5_6 =
933 pqi_aio_limit_to_bytes(
934 &buffer->aio_subpage.max_write_raid_5_6);
935
936 ctrl_info->max_write_raid_1_10_2drive =
937 pqi_aio_limit_to_bytes(
938 &buffer->aio_subpage.max_write_raid_1_10_2drive);
939
940 ctrl_info->max_write_raid_1_10_3drive =
941 pqi_aio_limit_to_bytes(
942 &buffer->aio_subpage.max_write_raid_1_10_3drive);
943
944error:
945 kfree(buffer);
946
947 return rc;
948}
949
950static int pqi_flush_cache(struct pqi_ctrl_info *ctrl_info,
951 enum bmic_flush_cache_shutdown_event shutdown_event)
952{
953 int rc;
954 struct bmic_flush_cache *flush_cache;
955
956 flush_cache = kzalloc_obj(*flush_cache);
957 if (!flush_cache)
958 return -ENOMEM;
959
960 flush_cache->shutdown_event = shutdown_event;
961
962 rc = pqi_send_ctrl_raid_request(ctrl_info, SA_FLUSH_CACHE, flush_cache,
963 sizeof(*flush_cache));
964
965 kfree(flush_cache);
966
967 return rc;
968}
969
970int pqi_csmi_smp_passthru(struct pqi_ctrl_info *ctrl_info,
971 struct bmic_csmi_smp_passthru_buffer *buffer, size_t buffer_length,
972 struct pqi_raid_error_info *error_info)
973{
974 return pqi_send_ctrl_raid_with_error(ctrl_info, BMIC_CSMI_PASSTHRU,
975 buffer, buffer_length, error_info);
976}
977
978#define PQI_FETCH_PTRAID_DATA (1 << 31)
979
980static int pqi_set_diag_rescan(struct pqi_ctrl_info *ctrl_info)
981{
982 int rc;
983 struct bmic_diag_options *diag;
984
985 diag = kzalloc_obj(*diag);
986 if (!diag)
987 return -ENOMEM;
988
989 rc = pqi_send_ctrl_raid_request(ctrl_info, BMIC_SENSE_DIAG_OPTIONS,
990 diag, sizeof(*diag));
991 if (rc)
992 goto out;
993
994 diag->options |= cpu_to_le32(PQI_FETCH_PTRAID_DATA);
995
996 rc = pqi_send_ctrl_raid_request(ctrl_info, BMIC_SET_DIAG_OPTIONS, diag,
997 sizeof(*diag));
998
999out:
1000 kfree(diag);
1001
1002 return rc;
1003}
1004
1005static inline int pqi_write_host_wellness(struct pqi_ctrl_info *ctrl_info,
1006 void *buffer, size_t buffer_length)
1007{
1008 return pqi_send_ctrl_raid_request(ctrl_info, BMIC_WRITE_HOST_WELLNESS,
1009 buffer, buffer_length);
1010}
1011
1012#pragma pack(1)
1013
1014struct bmic_host_wellness_driver_version {
1015 u8 start_tag[4];
1016 u8 driver_version_tag[2];
1017 __le16 driver_version_length;
1018 char driver_version[32];
1019 u8 dont_write_tag[2];
1020 u8 end_tag[2];
1021};
1022
1023#pragma pack()
1024
1025static int pqi_write_driver_version_to_host_wellness(
1026 struct pqi_ctrl_info *ctrl_info)
1027{
1028 int rc;
1029 struct bmic_host_wellness_driver_version *buffer;
1030 size_t buffer_length;
1031
1032 buffer_length = sizeof(*buffer);
1033
1034 buffer = kmalloc(buffer_length, GFP_KERNEL);
1035 if (!buffer)
1036 return -ENOMEM;
1037
1038 buffer->start_tag[0] = '<';
1039 buffer->start_tag[1] = 'H';
1040 buffer->start_tag[2] = 'W';
1041 buffer->start_tag[3] = '>';
1042 buffer->driver_version_tag[0] = 'D';
1043 buffer->driver_version_tag[1] = 'V';
1044 put_unaligned_le16(sizeof(buffer->driver_version),
1045 &buffer->driver_version_length);
1046 strscpy(buffer->driver_version, "Linux " DRIVER_VERSION,
1047 sizeof(buffer->driver_version));
1048 buffer->dont_write_tag[0] = 'D';
1049 buffer->dont_write_tag[1] = 'W';
1050 buffer->end_tag[0] = 'Z';
1051 buffer->end_tag[1] = 'Z';
1052
1053 rc = pqi_write_host_wellness(ctrl_info, buffer, buffer_length);
1054
1055 kfree(buffer);
1056
1057 return rc;
1058}
1059
1060#pragma pack(1)
1061
1062struct bmic_host_wellness_time {
1063 u8 start_tag[4];
1064 u8 time_tag[2];
1065 __le16 time_length;
1066 u8 time[8];
1067 u8 dont_write_tag[2];
1068 u8 end_tag[2];
1069};
1070
1071#pragma pack()
1072
1073static int pqi_write_current_time_to_host_wellness(
1074 struct pqi_ctrl_info *ctrl_info)
1075{
1076 int rc;
1077 struct bmic_host_wellness_time *buffer;
1078 size_t buffer_length;
1079 time64_t local_time;
1080 unsigned int year;
1081 struct tm tm;
1082
1083 buffer_length = sizeof(*buffer);
1084
1085 buffer = kmalloc(buffer_length, GFP_KERNEL);
1086 if (!buffer)
1087 return -ENOMEM;
1088
1089 buffer->start_tag[0] = '<';
1090 buffer->start_tag[1] = 'H';
1091 buffer->start_tag[2] = 'W';
1092 buffer->start_tag[3] = '>';
1093 buffer->time_tag[0] = 'T';
1094 buffer->time_tag[1] = 'D';
1095 put_unaligned_le16(sizeof(buffer->time),
1096 &buffer->time_length);
1097
1098 local_time = ktime_get_real_seconds();
1099 time64_to_tm(local_time, -sys_tz.tz_minuteswest * 60, &tm);
1100 year = tm.tm_year + 1900;
1101
1102 buffer->time[0] = bin2bcd(tm.tm_hour);
1103 buffer->time[1] = bin2bcd(tm.tm_min);
1104 buffer->time[2] = bin2bcd(tm.tm_sec);
1105 buffer->time[3] = 0;
1106 buffer->time[4] = bin2bcd(tm.tm_mon + 1);
1107 buffer->time[5] = bin2bcd(tm.tm_mday);
1108 buffer->time[6] = bin2bcd(year / 100);
1109 buffer->time[7] = bin2bcd(year % 100);
1110
1111 buffer->dont_write_tag[0] = 'D';
1112 buffer->dont_write_tag[1] = 'W';
1113 buffer->end_tag[0] = 'Z';
1114 buffer->end_tag[1] = 'Z';
1115
1116 rc = pqi_write_host_wellness(ctrl_info, buffer, buffer_length);
1117
1118 kfree(buffer);
1119
1120 return rc;
1121}
1122
1123#define PQI_UPDATE_TIME_WORK_INTERVAL (24UL * 60 * 60 * HZ)
1124
1125static void pqi_update_time_worker(struct work_struct *work)
1126{
1127 int rc;
1128 struct pqi_ctrl_info *ctrl_info;
1129
1130 ctrl_info = container_of(to_delayed_work(work), struct pqi_ctrl_info,
1131 update_time_work);
1132
1133 rc = pqi_write_current_time_to_host_wellness(ctrl_info);
1134 if (rc)
1135 dev_warn(&ctrl_info->pci_dev->dev,
1136 "error updating time on controller\n");
1137
1138 schedule_delayed_work(&ctrl_info->update_time_work,
1139 PQI_UPDATE_TIME_WORK_INTERVAL);
1140}
1141
1142static inline void pqi_schedule_update_time_worker(struct pqi_ctrl_info *ctrl_info)
1143{
1144 schedule_delayed_work(&ctrl_info->update_time_work, 0);
1145}
1146
1147static inline void pqi_cancel_update_time_worker(struct pqi_ctrl_info *ctrl_info)
1148{
1149 cancel_delayed_work_sync(&ctrl_info->update_time_work);
1150}
1151
1152static inline int pqi_report_luns(struct pqi_ctrl_info *ctrl_info, u8 cmd, void *buffer,
1153 size_t buffer_length)
1154{
1155 return pqi_send_ctrl_raid_request(ctrl_info, cmd, buffer, buffer_length);
1156}
1157
1158static int pqi_report_phys_logical_luns(struct pqi_ctrl_info *ctrl_info, u8 cmd, void **buffer)
1159{
1160 int rc;
1161 size_t lun_list_length;
1162 size_t lun_data_length;
1163 size_t new_lun_list_length;
1164 void *lun_data = NULL;
1165 struct report_lun_header *report_lun_header;
1166
1167 report_lun_header = kmalloc_obj(*report_lun_header);
1168 if (!report_lun_header) {
1169 rc = -ENOMEM;
1170 goto out;
1171 }
1172
1173 rc = pqi_report_luns(ctrl_info, cmd, report_lun_header, sizeof(*report_lun_header));
1174 if (rc)
1175 goto out;
1176
1177 lun_list_length = get_unaligned_be32(&report_lun_header->list_length);
1178
1179again:
1180 lun_data_length = sizeof(struct report_lun_header) + lun_list_length;
1181
1182 lun_data = kmalloc(lun_data_length, GFP_KERNEL);
1183 if (!lun_data) {
1184 rc = -ENOMEM;
1185 goto out;
1186 }
1187
1188 if (lun_list_length == 0) {
1189 memcpy(lun_data, report_lun_header, sizeof(*report_lun_header));
1190 goto out;
1191 }
1192
1193 rc = pqi_report_luns(ctrl_info, cmd, lun_data, lun_data_length);
1194 if (rc)
1195 goto out;
1196
1197 new_lun_list_length =
1198 get_unaligned_be32(&((struct report_lun_header *)lun_data)->list_length);
1199
1200 if (new_lun_list_length > lun_list_length) {
1201 lun_list_length = new_lun_list_length;
1202 kfree(lun_data);
1203 goto again;
1204 }
1205
1206out:
1207 kfree(report_lun_header);
1208
1209 if (rc) {
1210 kfree(lun_data);
1211 lun_data = NULL;
1212 }
1213
1214 *buffer = lun_data;
1215
1216 return rc;
1217}
1218
1219static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info, void **buffer)
1220{
1221 int rc;
1222 unsigned int i;
1223 u8 rpl_response_format;
1224 u32 num_physicals;
1225 void *rpl_list;
1226 struct report_lun_header *rpl_header;
1227 struct report_phys_lun_8byte_wwid_list *rpl_8byte_wwid_list;
1228 struct report_phys_lun_16byte_wwid_list *rpl_16byte_wwid_list;
1229
1230 rc = pqi_report_phys_logical_luns(ctrl_info, CISS_REPORT_PHYS, &rpl_list);
1231 if (rc)
1232 return rc;
1233
1234 if (ctrl_info->rpl_extended_format_4_5_supported) {
1235 rpl_header = rpl_list;
1236 rpl_response_format = rpl_header->flags & CISS_REPORT_PHYS_FLAG_EXTENDED_FORMAT_MASK;
1237 if (rpl_response_format == CISS_REPORT_PHYS_FLAG_EXTENDED_FORMAT_4) {
1238 *buffer = rpl_list;
1239 return 0;
1240 } else if (rpl_response_format != CISS_REPORT_PHYS_FLAG_EXTENDED_FORMAT_2) {
1241 dev_err(&ctrl_info->pci_dev->dev,
1242 "RPL returned unsupported data format %u\n",
1243 rpl_response_format);
1244 rc = -EINVAL;
1245 goto out_free_rpl_list;
1246 } else {
1247 dev_warn(&ctrl_info->pci_dev->dev,
1248 "RPL returned extended format 2 instead of 4\n");
1249 }
1250 }
1251
1252 rpl_8byte_wwid_list = rpl_list;
1253 num_physicals = get_unaligned_be32(&rpl_8byte_wwid_list->header.list_length) / sizeof(rpl_8byte_wwid_list->lun_entries[0]);
1254
1255 rpl_16byte_wwid_list = kmalloc_flex(*rpl_16byte_wwid_list, lun_entries,
1256 num_physicals);
1257 if (!rpl_16byte_wwid_list) {
1258 rc = -ENOMEM;
1259 goto out_free_rpl_list;
1260 }
1261
1262 put_unaligned_be32(num_physicals * sizeof(struct report_phys_lun_16byte_wwid),
1263 &rpl_16byte_wwid_list->header.list_length);
1264 rpl_16byte_wwid_list->header.flags = rpl_8byte_wwid_list->header.flags;
1265
1266 for (i = 0; i < num_physicals; i++) {
1267 memcpy(&rpl_16byte_wwid_list->lun_entries[i].lunid, &rpl_8byte_wwid_list->lun_entries[i].lunid, sizeof(rpl_8byte_wwid_list->lun_entries[i].lunid));
1268 memcpy(&rpl_16byte_wwid_list->lun_entries[i].wwid[0], &rpl_8byte_wwid_list->lun_entries[i].wwid, sizeof(rpl_8byte_wwid_list->lun_entries[i].wwid));
1269 memset(&rpl_16byte_wwid_list->lun_entries[i].wwid[8], 0, 8);
1270 rpl_16byte_wwid_list->lun_entries[i].device_type = rpl_8byte_wwid_list->lun_entries[i].device_type;
1271 rpl_16byte_wwid_list->lun_entries[i].device_flags = rpl_8byte_wwid_list->lun_entries[i].device_flags;
1272 rpl_16byte_wwid_list->lun_entries[i].lun_count = rpl_8byte_wwid_list->lun_entries[i].lun_count;
1273 rpl_16byte_wwid_list->lun_entries[i].redundant_paths = rpl_8byte_wwid_list->lun_entries[i].redundant_paths;
1274 rpl_16byte_wwid_list->lun_entries[i].aio_handle = rpl_8byte_wwid_list->lun_entries[i].aio_handle;
1275 }
1276
1277 kfree(rpl_8byte_wwid_list);
1278 *buffer = rpl_16byte_wwid_list;
1279
1280 return 0;
1281
1282out_free_rpl_list:
1283 kfree(rpl_list);
1284 return rc;
1285}
1286
1287static inline int pqi_report_logical_luns(struct pqi_ctrl_info *ctrl_info, void **buffer)
1288{
1289 return pqi_report_phys_logical_luns(ctrl_info, CISS_REPORT_LOG, buffer);
1290}
1291
1292static int pqi_get_device_lists(struct pqi_ctrl_info *ctrl_info,
1293 struct report_phys_lun_16byte_wwid_list **physdev_list,
1294 struct report_log_lun_list **logdev_list)
1295{
1296 int rc;
1297 size_t logdev_list_length;
1298 size_t logdev_data_length;
1299 struct report_log_lun_list *internal_logdev_list;
1300 struct report_log_lun_list *logdev_data;
1301 struct report_lun_header report_lun_header;
1302
1303 rc = pqi_report_phys_luns(ctrl_info, (void **)physdev_list);
1304 if (rc)
1305 dev_err(&ctrl_info->pci_dev->dev,
1306 "report physical LUNs failed\n");
1307
1308 rc = pqi_report_logical_luns(ctrl_info, (void **)logdev_list);
1309 if (rc)
1310 dev_err(&ctrl_info->pci_dev->dev,
1311 "report logical LUNs failed\n");
1312
1313 /*
1314 * Tack the controller itself onto the end of the logical device list
1315 * by adding a list entry that is all zeros.
1316 */
1317
1318 logdev_data = *logdev_list;
1319
1320 if (logdev_data) {
1321 logdev_list_length =
1322 get_unaligned_be32(&logdev_data->header.list_length);
1323 } else {
1324 memset(&report_lun_header, 0, sizeof(report_lun_header));
1325 logdev_data =
1326 (struct report_log_lun_list *)&report_lun_header;
1327 logdev_list_length = 0;
1328 }
1329
1330 logdev_data_length = sizeof(struct report_lun_header) +
1331 logdev_list_length;
1332
1333 internal_logdev_list = kmalloc(logdev_data_length +
1334 sizeof(struct report_log_lun), GFP_KERNEL);
1335 if (!internal_logdev_list) {
1336 kfree(*logdev_list);
1337 *logdev_list = NULL;
1338 return -ENOMEM;
1339 }
1340
1341 memcpy(internal_logdev_list, logdev_data, logdev_data_length);
1342 memset((u8 *)internal_logdev_list + logdev_data_length, 0,
1343 sizeof(struct report_log_lun));
1344 put_unaligned_be32(logdev_list_length +
1345 sizeof(struct report_log_lun),
1346 &internal_logdev_list->header.list_length);
1347
1348 kfree(*logdev_list);
1349 *logdev_list = internal_logdev_list;
1350
1351 return 0;
1352}
1353
1354static inline void pqi_set_bus_target_lun(struct pqi_scsi_dev *device,
1355 int bus, int target, int lun)
1356{
1357 device->bus = bus;
1358 device->target = target;
1359 device->lun = lun;
1360}
1361
1362static void pqi_assign_bus_target_lun(struct pqi_scsi_dev *device)
1363{
1364 u8 *scsi3addr;
1365 u32 lunid;
1366 int bus;
1367 int target;
1368 int lun;
1369
1370 scsi3addr = device->scsi3addr;
1371 lunid = get_unaligned_le32(scsi3addr);
1372
1373 if (pqi_is_hba_lunid(scsi3addr)) {
1374 /* The specified device is the controller. */
1375 pqi_set_bus_target_lun(device, PQI_HBA_BUS, 0, lunid & 0x3fff);
1376 device->target_lun_valid = true;
1377 return;
1378 }
1379
1380 if (pqi_is_logical_device(device)) {
1381 if (device->is_external_raid_device) {
1382 bus = PQI_EXTERNAL_RAID_VOLUME_BUS;
1383 target = (lunid >> 16) & 0x3fff;
1384 lun = lunid & 0xff;
1385 } else {
1386 bus = PQI_RAID_VOLUME_BUS;
1387 target = 0;
1388 lun = lunid & 0x3fff;
1389 }
1390 pqi_set_bus_target_lun(device, bus, target, lun);
1391 device->target_lun_valid = true;
1392 return;
1393 }
1394
1395 /*
1396 * Defer target and LUN assignment for non-controller physical devices
1397 * because the SAS transport layer will make these assignments later.
1398 */
1399 pqi_set_bus_target_lun(device, PQI_PHYSICAL_DEVICE_BUS, 0, 0);
1400}
1401
1402static void pqi_get_raid_level(struct pqi_ctrl_info *ctrl_info,
1403 struct pqi_scsi_dev *device)
1404{
1405 int rc;
1406 u8 raid_level;
1407 u8 *buffer;
1408
1409 raid_level = SA_RAID_UNKNOWN;
1410
1411 buffer = kmalloc(64, GFP_KERNEL);
1412 if (buffer) {
1413 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,
1414 VPD_PAGE | CISS_VPD_LV_DEVICE_GEOMETRY, buffer, 64);
1415 if (rc == 0) {
1416 raid_level = buffer[8];
1417 if (raid_level > SA_RAID_MAX)
1418 raid_level = SA_RAID_UNKNOWN;
1419 }
1420 kfree(buffer);
1421 }
1422
1423 device->raid_level = raid_level;
1424}
1425
1426static int pqi_validate_raid_map(struct pqi_ctrl_info *ctrl_info,
1427 struct pqi_scsi_dev *device, struct raid_map *raid_map)
1428{
1429 char *err_msg;
1430 u32 raid_map_size;
1431 u32 r5or6_blocks_per_row;
1432
1433 raid_map_size = get_unaligned_le32(&raid_map->structure_size);
1434
1435 if (raid_map_size < offsetof(struct raid_map, disk_data)) {
1436 err_msg = "RAID map too small";
1437 goto bad_raid_map;
1438 }
1439
1440 if (device->raid_level == SA_RAID_1) {
1441 if (get_unaligned_le16(&raid_map->layout_map_count) != 2) {
1442 err_msg = "invalid RAID-1 map";
1443 goto bad_raid_map;
1444 }
1445 } else if (device->raid_level == SA_RAID_TRIPLE) {
1446 if (get_unaligned_le16(&raid_map->layout_map_count) != 3) {
1447 err_msg = "invalid RAID-1(Triple) map";
1448 goto bad_raid_map;
1449 }
1450 } else if ((device->raid_level == SA_RAID_5 ||
1451 device->raid_level == SA_RAID_6) &&
1452 get_unaligned_le16(&raid_map->layout_map_count) > 1) {
1453 /* RAID 50/60 */
1454 r5or6_blocks_per_row =
1455 get_unaligned_le16(&raid_map->strip_size) *
1456 get_unaligned_le16(&raid_map->data_disks_per_row);
1457 if (r5or6_blocks_per_row == 0) {
1458 err_msg = "invalid RAID-5 or RAID-6 map";
1459 goto bad_raid_map;
1460 }
1461 }
1462
1463 return 0;
1464
1465bad_raid_map:
1466 dev_warn(&ctrl_info->pci_dev->dev,
1467 "logical device %08x%08x %s\n",
1468 *((u32 *)&device->scsi3addr),
1469 *((u32 *)&device->scsi3addr[4]), err_msg);
1470
1471 return -EINVAL;
1472}
1473
1474static int pqi_get_raid_map(struct pqi_ctrl_info *ctrl_info,
1475 struct pqi_scsi_dev *device)
1476{
1477 int rc;
1478 u32 raid_map_size;
1479 struct raid_map *raid_map;
1480
1481 raid_map = kmalloc_obj(*raid_map);
1482 if (!raid_map)
1483 return -ENOMEM;
1484
1485 rc = pqi_send_scsi_raid_request(ctrl_info, CISS_GET_RAID_MAP,
1486 device->scsi3addr, raid_map, sizeof(*raid_map), 0, NULL);
1487 if (rc)
1488 goto error;
1489
1490 raid_map_size = get_unaligned_le32(&raid_map->structure_size);
1491
1492 if (raid_map_size > sizeof(*raid_map)) {
1493
1494 kfree(raid_map);
1495
1496 raid_map = kmalloc(raid_map_size, GFP_KERNEL);
1497 if (!raid_map)
1498 return -ENOMEM;
1499
1500 rc = pqi_send_scsi_raid_request(ctrl_info, CISS_GET_RAID_MAP,
1501 device->scsi3addr, raid_map, raid_map_size, 0, NULL);
1502 if (rc)
1503 goto error;
1504
1505 if (get_unaligned_le32(&raid_map->structure_size)
1506 != raid_map_size) {
1507 dev_warn(&ctrl_info->pci_dev->dev,
1508 "requested %u bytes, received %u bytes\n",
1509 raid_map_size,
1510 get_unaligned_le32(&raid_map->structure_size));
1511 rc = -EINVAL;
1512 goto error;
1513 }
1514 }
1515
1516 rc = pqi_validate_raid_map(ctrl_info, device, raid_map);
1517 if (rc)
1518 goto error;
1519
1520 device->raid_io_stats = alloc_percpu(struct pqi_raid_io_stats);
1521 if (!device->raid_io_stats) {
1522 rc = -ENOMEM;
1523 goto error;
1524 }
1525
1526 device->raid_map = raid_map;
1527
1528 return 0;
1529
1530error:
1531 kfree(raid_map);
1532
1533 return rc;
1534}
1535
1536static void pqi_set_max_transfer_encrypted(struct pqi_ctrl_info *ctrl_info,
1537 struct pqi_scsi_dev *device)
1538{
1539 if (!ctrl_info->lv_drive_type_mix_valid) {
1540 device->max_transfer_encrypted = ~0;
1541 return;
1542 }
1543
1544 switch (LV_GET_DRIVE_TYPE_MIX(device->scsi3addr)) {
1545 case LV_DRIVE_TYPE_MIX_SAS_HDD_ONLY:
1546 case LV_DRIVE_TYPE_MIX_SATA_HDD_ONLY:
1547 case LV_DRIVE_TYPE_MIX_SAS_OR_SATA_SSD_ONLY:
1548 case LV_DRIVE_TYPE_MIX_SAS_SSD_ONLY:
1549 case LV_DRIVE_TYPE_MIX_SATA_SSD_ONLY:
1550 case LV_DRIVE_TYPE_MIX_SAS_ONLY:
1551 case LV_DRIVE_TYPE_MIX_SATA_ONLY:
1552 device->max_transfer_encrypted =
1553 ctrl_info->max_transfer_encrypted_sas_sata;
1554 break;
1555 case LV_DRIVE_TYPE_MIX_NVME_ONLY:
1556 device->max_transfer_encrypted =
1557 ctrl_info->max_transfer_encrypted_nvme;
1558 break;
1559 case LV_DRIVE_TYPE_MIX_UNKNOWN:
1560 case LV_DRIVE_TYPE_MIX_NO_RESTRICTION:
1561 default:
1562 device->max_transfer_encrypted =
1563 min(ctrl_info->max_transfer_encrypted_sas_sata,
1564 ctrl_info->max_transfer_encrypted_nvme);
1565 break;
1566 }
1567}
1568
1569static void pqi_get_raid_bypass_status(struct pqi_ctrl_info *ctrl_info,
1570 struct pqi_scsi_dev *device)
1571{
1572 int rc;
1573 u8 *buffer;
1574 u8 bypass_status;
1575
1576 buffer = kmalloc(64, GFP_KERNEL);
1577 if (!buffer)
1578 return;
1579
1580 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,
1581 VPD_PAGE | CISS_VPD_LV_BYPASS_STATUS, buffer, 64);
1582 if (rc)
1583 goto out;
1584
1585#define RAID_BYPASS_STATUS 4
1586#define RAID_BYPASS_CONFIGURED 0x1
1587#define RAID_BYPASS_ENABLED 0x2
1588
1589 bypass_status = buffer[RAID_BYPASS_STATUS];
1590 device->raid_bypass_configured =
1591 (bypass_status & RAID_BYPASS_CONFIGURED) != 0;
1592 if (device->raid_bypass_configured &&
1593 (bypass_status & RAID_BYPASS_ENABLED) &&
1594 pqi_get_raid_map(ctrl_info, device) == 0) {
1595 device->raid_bypass_enabled = true;
1596 if (get_unaligned_le16(&device->raid_map->flags) &
1597 RAID_MAP_ENCRYPTION_ENABLED)
1598 pqi_set_max_transfer_encrypted(ctrl_info, device);
1599 }
1600
1601out:
1602 kfree(buffer);
1603}
1604
1605/*
1606 * Use vendor-specific VPD to determine online/offline status of a volume.
1607 */
1608
1609static void pqi_get_volume_status(struct pqi_ctrl_info *ctrl_info,
1610 struct pqi_scsi_dev *device)
1611{
1612 int rc;
1613 size_t page_length;
1614 u8 volume_status = CISS_LV_STATUS_UNAVAILABLE;
1615 bool volume_offline = true;
1616 u32 volume_flags;
1617 struct ciss_vpd_logical_volume_status *vpd;
1618
1619 vpd = kmalloc_obj(*vpd);
1620 if (!vpd)
1621 goto no_buffer;
1622
1623 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,
1624 VPD_PAGE | CISS_VPD_LV_STATUS, vpd, sizeof(*vpd));
1625 if (rc)
1626 goto out;
1627
1628 if (vpd->page_code != CISS_VPD_LV_STATUS)
1629 goto out;
1630
1631 page_length = offsetof(struct ciss_vpd_logical_volume_status,
1632 volume_status) + vpd->page_length;
1633 if (page_length < sizeof(*vpd))
1634 goto out;
1635
1636 volume_status = vpd->volume_status;
1637 volume_flags = get_unaligned_be32(&vpd->flags);
1638 volume_offline = (volume_flags & CISS_LV_FLAGS_NO_HOST_IO) != 0;
1639
1640out:
1641 kfree(vpd);
1642no_buffer:
1643 device->volume_status = volume_status;
1644 device->volume_offline = volume_offline;
1645}
1646
1647#define PQI_DEVICE_NCQ_PRIO_SUPPORTED 0x01
1648#define PQI_DEVICE_PHY_MAP_SUPPORTED 0x10
1649#define PQI_DEVICE_ERASE_IN_PROGRESS 0x10
1650
1651static int pqi_get_physical_device_info(struct pqi_ctrl_info *ctrl_info,
1652 struct pqi_scsi_dev *device,
1653 struct bmic_identify_physical_device *id_phys)
1654{
1655 int rc;
1656
1657 memset(id_phys, 0, sizeof(*id_phys));
1658
1659 rc = pqi_identify_physical_device(ctrl_info, device,
1660 id_phys, sizeof(*id_phys));
1661 if (rc) {
1662 device->queue_depth = PQI_PHYSICAL_DISK_DEFAULT_MAX_QUEUE_DEPTH;
1663 return rc;
1664 }
1665
1666 scsi_sanitize_inquiry_string(&id_phys->model[0], 8);
1667 scsi_sanitize_inquiry_string(&id_phys->model[8], 16);
1668
1669 memcpy(device->vendor, &id_phys->model[0], sizeof(device->vendor));
1670 memcpy(device->model, &id_phys->model[8], sizeof(device->model));
1671
1672 device->box_index = id_phys->box_index;
1673 device->phys_box_on_bus = id_phys->phys_box_on_bus;
1674 device->phy_connected_dev_type = id_phys->phy_connected_dev_type[0];
1675 device->queue_depth =
1676 get_unaligned_le16(&id_phys->current_queue_depth_limit);
1677 device->active_path_index = id_phys->active_path_number;
1678 device->path_map = id_phys->redundant_path_present_map;
1679 memcpy(&device->box,
1680 &id_phys->alternate_paths_phys_box_on_port,
1681 sizeof(device->box));
1682 memcpy(&device->phys_connector,
1683 &id_phys->alternate_paths_phys_connector,
1684 sizeof(device->phys_connector));
1685 device->bay = id_phys->phys_bay_in_box;
1686 device->lun_count = id_phys->multi_lun_device_lun_count;
1687 if ((id_phys->even_more_flags & PQI_DEVICE_PHY_MAP_SUPPORTED) &&
1688 id_phys->phy_count)
1689 device->phy_id =
1690 id_phys->phy_to_phy_map[device->active_path_index];
1691 else
1692 device->phy_id = 0xFF;
1693
1694 device->ncq_prio_support =
1695 ((get_unaligned_le32(&id_phys->misc_drive_flags) >> 16) &
1696 PQI_DEVICE_NCQ_PRIO_SUPPORTED);
1697
1698 device->erase_in_progress = !!(get_unaligned_le16(&id_phys->extra_physical_drive_flags) & PQI_DEVICE_ERASE_IN_PROGRESS);
1699
1700 return 0;
1701}
1702
1703static int pqi_get_logical_device_info(struct pqi_ctrl_info *ctrl_info,
1704 struct pqi_scsi_dev *device)
1705{
1706 int rc;
1707 u8 *buffer;
1708
1709 buffer = kmalloc(64, GFP_KERNEL);
1710 if (!buffer)
1711 return -ENOMEM;
1712
1713 /* Send an inquiry to the device to see what it is. */
1714 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr, 0, buffer, 64);
1715 if (rc)
1716 goto out;
1717
1718 scsi_sanitize_inquiry_string(&buffer[8], 8);
1719 scsi_sanitize_inquiry_string(&buffer[16], 16);
1720
1721 device->devtype = buffer[0] & 0x1f;
1722 memcpy(device->vendor, &buffer[8], sizeof(device->vendor));
1723 memcpy(device->model, &buffer[16], sizeof(device->model));
1724
1725 if (device->devtype == TYPE_DISK) {
1726 if (device->is_external_raid_device) {
1727 device->raid_level = SA_RAID_UNKNOWN;
1728 device->volume_status = CISS_LV_OK;
1729 device->volume_offline = false;
1730 } else {
1731 pqi_get_raid_level(ctrl_info, device);
1732 pqi_get_raid_bypass_status(ctrl_info, device);
1733 pqi_get_volume_status(ctrl_info, device);
1734 }
1735 }
1736
1737out:
1738 kfree(buffer);
1739
1740 return rc;
1741}
1742
1743/*
1744 * Prevent adding drive to OS for some corner cases such as a drive
1745 * undergoing a sanitize (erase) operation. Some OSes will continue to poll
1746 * the drive until the sanitize completes, which can take hours,
1747 * resulting in long bootup delays. Commands such as TUR, READ_CAP
1748 * are allowed, but READ/WRITE cause check condition. So the OS
1749 * cannot check/read the partition table.
1750 * Note: devices that have completed sanitize must be re-enabled
1751 * using the management utility.
1752 */
1753static inline bool pqi_keep_device_offline(struct pqi_scsi_dev *device)
1754{
1755 return device->erase_in_progress;
1756}
1757
1758static int pqi_get_device_info_phys_logical(struct pqi_ctrl_info *ctrl_info,
1759 struct pqi_scsi_dev *device,
1760 struct bmic_identify_physical_device *id_phys)
1761{
1762 int rc;
1763
1764 if (device->is_expander_smp_device)
1765 return 0;
1766
1767 if (pqi_is_logical_device(device))
1768 rc = pqi_get_logical_device_info(ctrl_info, device);
1769 else
1770 rc = pqi_get_physical_device_info(ctrl_info, device, id_phys);
1771
1772 return rc;
1773}
1774
1775static int pqi_get_device_info(struct pqi_ctrl_info *ctrl_info,
1776 struct pqi_scsi_dev *device,
1777 struct bmic_identify_physical_device *id_phys)
1778{
1779 int rc;
1780
1781 rc = pqi_get_device_info_phys_logical(ctrl_info, device, id_phys);
1782
1783 if (rc == 0 && device->lun_count == 0)
1784 device->lun_count = 1;
1785
1786 return rc;
1787}
1788
1789static void pqi_show_volume_status(struct pqi_ctrl_info *ctrl_info,
1790 struct pqi_scsi_dev *device)
1791{
1792 char *status;
1793 static const char unknown_state_str[] =
1794 "Volume is in an unknown state (%u)";
1795 char unknown_state_buffer[sizeof(unknown_state_str) + 10];
1796
1797 switch (device->volume_status) {
1798 case CISS_LV_OK:
1799 status = "Volume online";
1800 break;
1801 case CISS_LV_FAILED:
1802 status = "Volume failed";
1803 break;
1804 case CISS_LV_NOT_CONFIGURED:
1805 status = "Volume not configured";
1806 break;
1807 case CISS_LV_DEGRADED:
1808 status = "Volume degraded";
1809 break;
1810 case CISS_LV_READY_FOR_RECOVERY:
1811 status = "Volume ready for recovery operation";
1812 break;
1813 case CISS_LV_UNDERGOING_RECOVERY:
1814 status = "Volume undergoing recovery";
1815 break;
1816 case CISS_LV_WRONG_PHYSICAL_DRIVE_REPLACED:
1817 status = "Wrong physical drive was replaced";
1818 break;
1819 case CISS_LV_PHYSICAL_DRIVE_CONNECTION_PROBLEM:
1820 status = "A physical drive not properly connected";
1821 break;
1822 case CISS_LV_HARDWARE_OVERHEATING:
1823 status = "Hardware is overheating";
1824 break;
1825 case CISS_LV_HARDWARE_HAS_OVERHEATED:
1826 status = "Hardware has overheated";
1827 break;
1828 case CISS_LV_UNDERGOING_EXPANSION:
1829 status = "Volume undergoing expansion";
1830 break;
1831 case CISS_LV_NOT_AVAILABLE:
1832 status = "Volume waiting for transforming volume";
1833 break;
1834 case CISS_LV_QUEUED_FOR_EXPANSION:
1835 status = "Volume queued for expansion";
1836 break;
1837 case CISS_LV_DISABLED_SCSI_ID_CONFLICT:
1838 status = "Volume disabled due to SCSI ID conflict";
1839 break;
1840 case CISS_LV_EJECTED:
1841 status = "Volume has been ejected";
1842 break;
1843 case CISS_LV_UNDERGOING_ERASE:
1844 status = "Volume undergoing background erase";
1845 break;
1846 case CISS_LV_READY_FOR_PREDICTIVE_SPARE_REBUILD:
1847 status = "Volume ready for predictive spare rebuild";
1848 break;
1849 case CISS_LV_UNDERGOING_RPI:
1850 status = "Volume undergoing rapid parity initialization";
1851 break;
1852 case CISS_LV_PENDING_RPI:
1853 status = "Volume queued for rapid parity initialization";
1854 break;
1855 case CISS_LV_ENCRYPTED_NO_KEY:
1856 status = "Encrypted volume inaccessible - key not present";
1857 break;
1858 case CISS_LV_UNDERGOING_ENCRYPTION:
1859 status = "Volume undergoing encryption process";
1860 break;
1861 case CISS_LV_UNDERGOING_ENCRYPTION_REKEYING:
1862 status = "Volume undergoing encryption re-keying process";
1863 break;
1864 case CISS_LV_ENCRYPTED_IN_NON_ENCRYPTED_CONTROLLER:
1865 status = "Volume encrypted but encryption is disabled";
1866 break;
1867 case CISS_LV_PENDING_ENCRYPTION:
1868 status = "Volume pending migration to encrypted state";
1869 break;
1870 case CISS_LV_PENDING_ENCRYPTION_REKEYING:
1871 status = "Volume pending encryption rekeying";
1872 break;
1873 case CISS_LV_NOT_SUPPORTED:
1874 status = "Volume not supported on this controller";
1875 break;
1876 case CISS_LV_STATUS_UNAVAILABLE:
1877 status = "Volume status not available";
1878 break;
1879 default:
1880 snprintf(unknown_state_buffer, sizeof(unknown_state_buffer),
1881 unknown_state_str, device->volume_status);
1882 status = unknown_state_buffer;
1883 break;
1884 }
1885
1886 dev_info(&ctrl_info->pci_dev->dev,
1887 "scsi %d:%d:%d:%d %s\n",
1888 ctrl_info->scsi_host->host_no,
1889 device->bus, device->target, device->lun, status);
1890}
1891
1892static void pqi_rescan_worker(struct work_struct *work)
1893{
1894 struct pqi_ctrl_info *ctrl_info;
1895
1896 ctrl_info = container_of(to_delayed_work(work), struct pqi_ctrl_info,
1897 rescan_work);
1898
1899 pqi_scan_scsi_devices(ctrl_info);
1900}
1901
1902static int pqi_add_device(struct pqi_ctrl_info *ctrl_info,
1903 struct pqi_scsi_dev *device)
1904{
1905 int rc;
1906
1907 if (pqi_is_logical_device(device))
1908 rc = scsi_add_device(ctrl_info->scsi_host, device->bus,
1909 device->target, device->lun);
1910 else
1911 rc = pqi_add_sas_device(ctrl_info->sas_host, device);
1912
1913 return rc;
1914}
1915
1916#define PQI_REMOVE_DEVICE_PENDING_IO_TIMEOUT_MSECS (20 * 1000)
1917
1918static inline void pqi_remove_device(struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev *device)
1919{
1920 int rc;
1921 int lun;
1922
1923 for (lun = 0; lun < device->lun_count; lun++) {
1924 rc = pqi_device_wait_for_pending_io(ctrl_info, device, lun,
1925 PQI_REMOVE_DEVICE_PENDING_IO_TIMEOUT_MSECS);
1926 if (rc)
1927 dev_err(&ctrl_info->pci_dev->dev,
1928 "scsi %d:%d:%d:%d removing device with %d outstanding command(s)\n",
1929 ctrl_info->scsi_host->host_no, device->bus,
1930 device->target, lun,
1931 atomic_read(&device->scsi_cmds_outstanding[lun]));
1932 }
1933
1934 if (pqi_is_logical_device(device))
1935 scsi_remove_device(device->sdev);
1936 else
1937 pqi_remove_sas_device(device);
1938
1939 pqi_device_remove_start(device);
1940}
1941
1942/* Assumes the SCSI device list lock is held. */
1943
1944static struct pqi_scsi_dev *pqi_find_scsi_dev(struct pqi_ctrl_info *ctrl_info,
1945 int bus, int target, int lun)
1946{
1947 struct pqi_scsi_dev *device;
1948
1949 list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry)
1950 if (device->bus == bus && device->target == target && device->lun == lun)
1951 return device;
1952
1953 return NULL;
1954}
1955
1956static inline bool pqi_device_equal(struct pqi_scsi_dev *dev1, struct pqi_scsi_dev *dev2)
1957{
1958 if (dev1->is_physical_device != dev2->is_physical_device)
1959 return false;
1960
1961 if (dev1->is_physical_device)
1962 return memcmp(dev1->wwid, dev2->wwid, sizeof(dev1->wwid)) == 0;
1963
1964 return memcmp(dev1->volume_id, dev2->volume_id, sizeof(dev1->volume_id)) == 0;
1965}
1966
1967enum pqi_find_result {
1968 DEVICE_NOT_FOUND,
1969 DEVICE_CHANGED,
1970 DEVICE_SAME,
1971};
1972
1973static enum pqi_find_result pqi_scsi_find_entry(struct pqi_ctrl_info *ctrl_info,
1974 struct pqi_scsi_dev *device_to_find, struct pqi_scsi_dev **matching_device)
1975{
1976 struct pqi_scsi_dev *device;
1977
1978 list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry) {
1979 if (pqi_scsi3addr_equal(device_to_find->scsi3addr, device->scsi3addr)) {
1980 *matching_device = device;
1981 if (pqi_device_equal(device_to_find, device)) {
1982 if (device_to_find->volume_offline)
1983 return DEVICE_CHANGED;
1984 return DEVICE_SAME;
1985 }
1986 return DEVICE_CHANGED;
1987 }
1988 }
1989
1990 return DEVICE_NOT_FOUND;
1991}
1992
1993static inline const char *pqi_device_type(struct pqi_scsi_dev *device)
1994{
1995 if (device->is_expander_smp_device)
1996 return "Enclosure SMP ";
1997
1998 return scsi_device_type(device->devtype);
1999}
2000
2001#define PQI_DEV_INFO_BUFFER_LENGTH 128
2002
2003static void pqi_dev_info(struct pqi_ctrl_info *ctrl_info,
2004 char *action, struct pqi_scsi_dev *device)
2005{
2006 ssize_t count;
2007 char buffer[PQI_DEV_INFO_BUFFER_LENGTH];
2008
2009 count = scnprintf(buffer, PQI_DEV_INFO_BUFFER_LENGTH,
2010 "%d:%d:", ctrl_info->scsi_host->host_no, device->bus);
2011
2012 if (device->target_lun_valid)
2013 count += scnprintf(buffer + count,
2014 PQI_DEV_INFO_BUFFER_LENGTH - count,
2015 "%d:%d",
2016 device->target,
2017 device->lun);
2018 else
2019 count += scnprintf(buffer + count,
2020 PQI_DEV_INFO_BUFFER_LENGTH - count,
2021 "-:-");
2022
2023 if (pqi_is_logical_device(device)) {
2024 count += scnprintf(buffer + count,
2025 PQI_DEV_INFO_BUFFER_LENGTH - count,
2026 " %08x%08x",
2027 *((u32 *)&device->scsi3addr),
2028 *((u32 *)&device->scsi3addr[4]));
2029 } else if (ctrl_info->rpl_extended_format_4_5_supported) {
2030 if (device->device_type == SA_DEVICE_TYPE_NVME)
2031 count += scnprintf(buffer + count,
2032 PQI_DEV_INFO_BUFFER_LENGTH - count,
2033 " %016llx%016llx",
2034 get_unaligned_be64(&device->wwid[0]),
2035 get_unaligned_be64(&device->wwid[8]));
2036 else
2037 count += scnprintf(buffer + count,
2038 PQI_DEV_INFO_BUFFER_LENGTH - count,
2039 " %016llx",
2040 get_unaligned_be64(&device->wwid[0]));
2041 } else {
2042 count += scnprintf(buffer + count,
2043 PQI_DEV_INFO_BUFFER_LENGTH - count,
2044 " %016llx",
2045 get_unaligned_be64(&device->wwid[0]));
2046 }
2047
2048
2049 count += scnprintf(buffer + count, PQI_DEV_INFO_BUFFER_LENGTH - count,
2050 " %s %.8s %.16s ",
2051 pqi_device_type(device),
2052 device->vendor,
2053 device->model);
2054
2055 if (pqi_is_logical_device(device)) {
2056 if (device->devtype == TYPE_DISK)
2057 count += scnprintf(buffer + count,
2058 PQI_DEV_INFO_BUFFER_LENGTH - count,
2059 "SSDSmartPathCap%c En%c %-12s",
2060 device->raid_bypass_configured ? '+' : '-',
2061 device->raid_bypass_enabled ? '+' : '-',
2062 pqi_raid_level_to_string(device->raid_level));
2063 } else {
2064 count += scnprintf(buffer + count,
2065 PQI_DEV_INFO_BUFFER_LENGTH - count,
2066 "AIO%c", device->aio_enabled ? '+' : '-');
2067 if (device->devtype == TYPE_DISK ||
2068 device->devtype == TYPE_ZBC)
2069 count += scnprintf(buffer + count,
2070 PQI_DEV_INFO_BUFFER_LENGTH - count,
2071 " qd=%-6d", device->queue_depth);
2072 }
2073
2074 dev_info(&ctrl_info->pci_dev->dev, "%s %s\n", action, buffer);
2075}
2076
2077static bool pqi_raid_maps_equal(struct raid_map *raid_map1, struct raid_map *raid_map2)
2078{
2079 u32 raid_map1_size;
2080 u32 raid_map2_size;
2081
2082 if (raid_map1 == NULL || raid_map2 == NULL)
2083 return raid_map1 == raid_map2;
2084
2085 raid_map1_size = get_unaligned_le32(&raid_map1->structure_size);
2086 raid_map2_size = get_unaligned_le32(&raid_map2->structure_size);
2087
2088 if (raid_map1_size != raid_map2_size)
2089 return false;
2090
2091 return memcmp(raid_map1, raid_map2, raid_map1_size) == 0;
2092}
2093
2094/* Assumes the SCSI device list lock is held. */
2095
2096static void pqi_scsi_update_device(struct pqi_ctrl_info *ctrl_info,
2097 struct pqi_scsi_dev *existing_device, struct pqi_scsi_dev *new_device)
2098{
2099 existing_device->device_type = new_device->device_type;
2100 existing_device->bus = new_device->bus;
2101 if (new_device->target_lun_valid) {
2102 existing_device->target = new_device->target;
2103 existing_device->lun = new_device->lun;
2104 existing_device->target_lun_valid = true;
2105 }
2106
2107 /* By definition, the scsi3addr and wwid fields are already the same. */
2108
2109 existing_device->is_physical_device = new_device->is_physical_device;
2110 memcpy(existing_device->vendor, new_device->vendor, sizeof(existing_device->vendor));
2111 memcpy(existing_device->model, new_device->model, sizeof(existing_device->model));
2112 existing_device->sas_address = new_device->sas_address;
2113 existing_device->queue_depth = new_device->queue_depth;
2114 existing_device->device_offline = false;
2115 existing_device->lun_count = new_device->lun_count;
2116
2117 if (pqi_is_logical_device(existing_device)) {
2118 existing_device->is_external_raid_device = new_device->is_external_raid_device;
2119
2120 if (existing_device->devtype == TYPE_DISK) {
2121 existing_device->raid_level = new_device->raid_level;
2122 existing_device->volume_status = new_device->volume_status;
2123 memset(existing_device->next_bypass_group, 0, sizeof(existing_device->next_bypass_group));
2124 if (!pqi_raid_maps_equal(existing_device->raid_map, new_device->raid_map)) {
2125 kfree(existing_device->raid_map);
2126 existing_device->raid_map = new_device->raid_map;
2127 /* To prevent this from being freed later. */
2128 new_device->raid_map = NULL;
2129 }
2130 if (new_device->raid_bypass_enabled && existing_device->raid_io_stats == NULL) {
2131 existing_device->raid_io_stats = new_device->raid_io_stats;
2132 new_device->raid_io_stats = NULL;
2133 }
2134 existing_device->raid_bypass_configured = new_device->raid_bypass_configured;
2135 existing_device->raid_bypass_enabled = new_device->raid_bypass_enabled;
2136 }
2137 } else {
2138 existing_device->aio_enabled = new_device->aio_enabled;
2139 existing_device->aio_handle = new_device->aio_handle;
2140 existing_device->is_expander_smp_device = new_device->is_expander_smp_device;
2141 existing_device->active_path_index = new_device->active_path_index;
2142 existing_device->phy_id = new_device->phy_id;
2143 existing_device->path_map = new_device->path_map;
2144 existing_device->bay = new_device->bay;
2145 existing_device->box_index = new_device->box_index;
2146 existing_device->phys_box_on_bus = new_device->phys_box_on_bus;
2147 existing_device->phy_connected_dev_type = new_device->phy_connected_dev_type;
2148 memcpy(existing_device->box, new_device->box, sizeof(existing_device->box));
2149 memcpy(existing_device->phys_connector, new_device->phys_connector, sizeof(existing_device->phys_connector));
2150 }
2151}
2152
2153static inline void pqi_free_device(struct pqi_scsi_dev *device)
2154{
2155 if (device) {
2156 free_percpu(device->raid_io_stats);
2157 kfree(device->raid_map);
2158 kfree(device);
2159 }
2160}
2161
2162/*
2163 * Called when exposing a new device to the OS fails in order to re-adjust
2164 * our internal SCSI device list to match the SCSI ML's view.
2165 */
2166
2167static inline void pqi_fixup_botched_add(struct pqi_ctrl_info *ctrl_info,
2168 struct pqi_scsi_dev *device)
2169{
2170 unsigned long flags;
2171
2172 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
2173 list_del(&device->scsi_device_list_entry);
2174 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
2175
2176 /* Allow the device structure to be freed later. */
2177 device->keep_device = false;
2178}
2179
2180static inline bool pqi_is_device_added(struct pqi_scsi_dev *device)
2181{
2182 if (device->is_expander_smp_device)
2183 return device->sas_port != NULL;
2184
2185 return device->sdev != NULL;
2186}
2187
2188static inline void pqi_init_device_tmf_work(struct pqi_scsi_dev *device)
2189{
2190 unsigned int lun;
2191 struct pqi_tmf_work *tmf_work;
2192
2193 for (lun = 0, tmf_work = device->tmf_work; lun < PQI_MAX_LUNS_PER_DEVICE; lun++, tmf_work++)
2194 INIT_WORK(&tmf_work->work_struct, pqi_tmf_worker);
2195}
2196
2197static inline bool pqi_volume_rescan_needed(struct pqi_scsi_dev *device)
2198{
2199 if (pqi_device_in_remove(device))
2200 return false;
2201
2202 if (device->sdev == NULL)
2203 return false;
2204
2205 if (!scsi_device_online(device->sdev))
2206 return false;
2207
2208 return device->rescan;
2209}
2210
2211static void pqi_update_device_list(struct pqi_ctrl_info *ctrl_info,
2212 struct pqi_scsi_dev *new_device_list[], unsigned int num_new_devices)
2213{
2214 int rc;
2215 unsigned int i;
2216 unsigned long flags;
2217 enum pqi_find_result find_result;
2218 struct pqi_scsi_dev *device;
2219 struct pqi_scsi_dev *next;
2220 struct pqi_scsi_dev *matching_device;
2221 LIST_HEAD(add_list);
2222 LIST_HEAD(delete_list);
2223
2224 /*
2225 * The idea here is to do as little work as possible while holding the
2226 * spinlock. That's why we go to great pains to defer anything other
2227 * than updating the internal device list until after we release the
2228 * spinlock.
2229 */
2230
2231 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
2232
2233 /* Assume that all devices in the existing list have gone away. */
2234 list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry)
2235 device->device_gone = true;
2236
2237 for (i = 0; i < num_new_devices; i++) {
2238 device = new_device_list[i];
2239
2240 find_result = pqi_scsi_find_entry(ctrl_info, device,
2241 &matching_device);
2242
2243 switch (find_result) {
2244 case DEVICE_SAME:
2245 /*
2246 * The newly found device is already in the existing
2247 * device list.
2248 */
2249 device->new_device = false;
2250 matching_device->device_gone = false;
2251 pqi_scsi_update_device(ctrl_info, matching_device, device);
2252 break;
2253 case DEVICE_NOT_FOUND:
2254 /*
2255 * The newly found device is NOT in the existing device
2256 * list.
2257 */
2258 device->new_device = true;
2259 break;
2260 case DEVICE_CHANGED:
2261 /*
2262 * The original device has gone away and we need to add
2263 * the new device.
2264 */
2265 device->new_device = true;
2266 break;
2267 }
2268 }
2269
2270 /* Process all devices that have gone away. */
2271 list_for_each_entry_safe(device, next, &ctrl_info->scsi_device_list,
2272 scsi_device_list_entry) {
2273 if (device->device_gone) {
2274 list_del(&device->scsi_device_list_entry);
2275 list_add_tail(&device->delete_list_entry, &delete_list);
2276 }
2277 }
2278
2279 /* Process all new devices. */
2280 for (i = 0; i < num_new_devices; i++) {
2281 device = new_device_list[i];
2282 if (!device->new_device)
2283 continue;
2284 if (device->volume_offline)
2285 continue;
2286 list_add_tail(&device->scsi_device_list_entry,
2287 &ctrl_info->scsi_device_list);
2288 list_add_tail(&device->add_list_entry, &add_list);
2289 /* To prevent this device structure from being freed later. */
2290 device->keep_device = true;
2291 pqi_init_device_tmf_work(device);
2292 }
2293
2294 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
2295
2296 /*
2297 * If OFA is in progress and there are devices that need to be deleted,
2298 * allow any pending reset operations to continue and unblock any SCSI
2299 * requests before removal.
2300 */
2301 if (pqi_ofa_in_progress(ctrl_info)) {
2302 list_for_each_entry_safe(device, next, &delete_list, delete_list_entry)
2303 if (pqi_is_device_added(device))
2304 pqi_device_remove_start(device);
2305 pqi_ctrl_unblock_device_reset(ctrl_info);
2306 pqi_scsi_unblock_requests(ctrl_info);
2307 }
2308
2309 /* Remove all devices that have gone away. */
2310 list_for_each_entry_safe(device, next, &delete_list, delete_list_entry) {
2311 if (device->volume_offline) {
2312 pqi_dev_info(ctrl_info, "offline", device);
2313 pqi_show_volume_status(ctrl_info, device);
2314 } else {
2315 pqi_dev_info(ctrl_info, "removed", device);
2316 }
2317 if (pqi_is_device_added(device))
2318 pqi_remove_device(ctrl_info, device);
2319 list_del(&device->delete_list_entry);
2320 pqi_free_device(device);
2321 }
2322
2323 /*
2324 * Notify the SML of any existing device changes such as;
2325 * queue depth, device size.
2326 */
2327 list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry) {
2328 /*
2329 * Check for queue depth change.
2330 */
2331 if (device->sdev && device->queue_depth != device->advertised_queue_depth) {
2332 device->advertised_queue_depth = device->queue_depth;
2333 scsi_change_queue_depth(device->sdev, device->advertised_queue_depth);
2334 }
2335 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
2336 /*
2337 * Check for changes in the device, such as size.
2338 */
2339 if (pqi_volume_rescan_needed(device)) {
2340 device->rescan = false;
2341 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
2342 scsi_rescan_device(device->sdev);
2343 } else {
2344 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
2345 }
2346 }
2347
2348 /* Expose any new devices. */
2349 list_for_each_entry_safe(device, next, &add_list, add_list_entry) {
2350 if (!pqi_is_device_added(device)) {
2351 rc = pqi_add_device(ctrl_info, device);
2352 if (rc == 0) {
2353 pqi_dev_info(ctrl_info, "added", device);
2354 } else {
2355 dev_warn(&ctrl_info->pci_dev->dev,
2356 "scsi %d:%d:%d:%d addition failed, device not added\n",
2357 ctrl_info->scsi_host->host_no,
2358 device->bus, device->target,
2359 device->lun);
2360 pqi_fixup_botched_add(ctrl_info, device);
2361 }
2362 }
2363 }
2364
2365}
2366
2367static inline bool pqi_is_supported_device(struct pqi_scsi_dev *device)
2368{
2369 /*
2370 * Only support the HBA controller itself as a RAID
2371 * controller. If it's a RAID controller other than
2372 * the HBA itself (an external RAID controller, for
2373 * example), we don't support it.
2374 */
2375 if (device->device_type == SA_DEVICE_TYPE_CONTROLLER &&
2376 !pqi_is_hba_lunid(device->scsi3addr))
2377 return false;
2378
2379 return true;
2380}
2381
2382static inline bool pqi_skip_device(u8 *scsi3addr)
2383{
2384 /* Ignore all masked devices. */
2385 if (MASKED_DEVICE(scsi3addr))
2386 return true;
2387
2388 return false;
2389}
2390
2391static inline void pqi_mask_device(u8 *scsi3addr)
2392{
2393 scsi3addr[3] |= 0xc0;
2394}
2395
2396static inline bool pqi_expose_device(struct pqi_scsi_dev *device)
2397{
2398 return !device->is_physical_device || !pqi_skip_device(device->scsi3addr);
2399}
2400
2401static int pqi_update_scsi_devices(struct pqi_ctrl_info *ctrl_info)
2402{
2403 int i;
2404 int rc;
2405 LIST_HEAD(new_device_list_head);
2406 struct report_phys_lun_16byte_wwid_list *physdev_list = NULL;
2407 struct report_log_lun_list *logdev_list = NULL;
2408 struct report_phys_lun_16byte_wwid *phys_lun;
2409 struct report_log_lun *log_lun;
2410 struct bmic_identify_physical_device *id_phys = NULL;
2411 u32 num_physicals;
2412 u32 num_logicals;
2413 struct pqi_scsi_dev **new_device_list = NULL;
2414 struct pqi_scsi_dev *device;
2415 struct pqi_scsi_dev *next;
2416 unsigned int num_new_devices;
2417 unsigned int num_valid_devices;
2418 bool is_physical_device;
2419 u8 *scsi3addr;
2420 unsigned int physical_index;
2421 unsigned int logical_index;
2422 static char *out_of_memory_msg =
2423 "failed to allocate memory, device discovery stopped";
2424
2425 rc = pqi_get_device_lists(ctrl_info, &physdev_list, &logdev_list);
2426 if (rc)
2427 goto out;
2428
2429 if (physdev_list)
2430 num_physicals =
2431 get_unaligned_be32(&physdev_list->header.list_length)
2432 / sizeof(physdev_list->lun_entries[0]);
2433 else
2434 num_physicals = 0;
2435
2436 if (logdev_list)
2437 num_logicals =
2438 get_unaligned_be32(&logdev_list->header.list_length)
2439 / sizeof(logdev_list->lun_entries[0]);
2440 else
2441 num_logicals = 0;
2442
2443 if (num_physicals) {
2444 /*
2445 * We need this buffer for calls to pqi_get_physical_disk_info()
2446 * below. We allocate it here instead of inside
2447 * pqi_get_physical_disk_info() because it's a fairly large
2448 * buffer.
2449 */
2450 id_phys = kmalloc_obj(*id_phys);
2451 if (!id_phys) {
2452 dev_warn(&ctrl_info->pci_dev->dev, "%s\n",
2453 out_of_memory_msg);
2454 rc = -ENOMEM;
2455 goto out;
2456 }
2457
2458 if (pqi_hide_vsep) {
2459 for (i = num_physicals - 1; i >= 0; i--) {
2460 phys_lun = &physdev_list->lun_entries[i];
2461 if (CISS_GET_DRIVE_NUMBER(phys_lun->lunid) == PQI_VSEP_CISS_BTL) {
2462 pqi_mask_device(phys_lun->lunid);
2463 break;
2464 }
2465 }
2466 }
2467 }
2468
2469 if (num_logicals &&
2470 (logdev_list->header.flags & CISS_REPORT_LOG_FLAG_DRIVE_TYPE_MIX))
2471 ctrl_info->lv_drive_type_mix_valid = true;
2472
2473 num_new_devices = num_physicals + num_logicals;
2474
2475 new_device_list = kmalloc_objs(*new_device_list, num_new_devices);
2476 if (!new_device_list) {
2477 dev_warn(&ctrl_info->pci_dev->dev, "%s\n", out_of_memory_msg);
2478 rc = -ENOMEM;
2479 goto out;
2480 }
2481
2482 for (i = 0; i < num_new_devices; i++) {
2483 device = kzalloc_obj(*device);
2484 if (!device) {
2485 dev_warn(&ctrl_info->pci_dev->dev, "%s\n",
2486 out_of_memory_msg);
2487 rc = -ENOMEM;
2488 goto out;
2489 }
2490 list_add_tail(&device->new_device_list_entry,
2491 &new_device_list_head);
2492 }
2493
2494 device = NULL;
2495 num_valid_devices = 0;
2496 physical_index = 0;
2497 logical_index = 0;
2498
2499 for (i = 0; i < num_new_devices; i++) {
2500
2501 if ((!pqi_expose_ld_first && i < num_physicals) ||
2502 (pqi_expose_ld_first && i >= num_logicals)) {
2503 is_physical_device = true;
2504 phys_lun = &physdev_list->lun_entries[physical_index++];
2505 log_lun = NULL;
2506 scsi3addr = phys_lun->lunid;
2507 } else {
2508 is_physical_device = false;
2509 phys_lun = NULL;
2510 log_lun = &logdev_list->lun_entries[logical_index++];
2511 scsi3addr = log_lun->lunid;
2512 }
2513
2514 if (is_physical_device && pqi_skip_device(scsi3addr))
2515 continue;
2516
2517 if (device)
2518 device = list_next_entry(device, new_device_list_entry);
2519 else
2520 device = list_first_entry(&new_device_list_head,
2521 struct pqi_scsi_dev, new_device_list_entry);
2522
2523 memcpy(device->scsi3addr, scsi3addr, sizeof(device->scsi3addr));
2524 device->is_physical_device = is_physical_device;
2525 if (is_physical_device) {
2526 device->device_type = phys_lun->device_type;
2527 if (device->device_type == SA_DEVICE_TYPE_EXPANDER_SMP)
2528 device->is_expander_smp_device = true;
2529 } else {
2530 device->is_external_raid_device =
2531 pqi_is_external_raid_addr(scsi3addr);
2532 }
2533
2534 if (!pqi_is_supported_device(device))
2535 continue;
2536
2537 /* Gather information about the device. */
2538 rc = pqi_get_device_info(ctrl_info, device, id_phys);
2539 if (rc == -ENOMEM) {
2540 dev_warn(&ctrl_info->pci_dev->dev, "%s\n",
2541 out_of_memory_msg);
2542 goto out;
2543 }
2544 if (rc) {
2545 if (device->is_physical_device)
2546 dev_warn(&ctrl_info->pci_dev->dev,
2547 "obtaining device info failed, skipping physical device %016llx%016llx\n",
2548 get_unaligned_be64(&phys_lun->wwid[0]),
2549 get_unaligned_be64(&phys_lun->wwid[8]));
2550 else
2551 dev_warn(&ctrl_info->pci_dev->dev,
2552 "obtaining device info failed, skipping logical device %08x%08x\n",
2553 *((u32 *)&device->scsi3addr),
2554 *((u32 *)&device->scsi3addr[4]));
2555 rc = 0;
2556 continue;
2557 }
2558
2559 /* Do not present disks that the OS cannot fully probe. */
2560 if (pqi_keep_device_offline(device))
2561 continue;
2562
2563 pqi_assign_bus_target_lun(device);
2564
2565 if (device->is_physical_device) {
2566 memcpy(device->wwid, phys_lun->wwid, sizeof(device->wwid));
2567 if ((phys_lun->device_flags &
2568 CISS_REPORT_PHYS_DEV_FLAG_AIO_ENABLED) &&
2569 phys_lun->aio_handle) {
2570 device->aio_enabled = true;
2571 device->aio_handle =
2572 phys_lun->aio_handle;
2573 }
2574 } else {
2575 memcpy(device->volume_id, log_lun->volume_id,
2576 sizeof(device->volume_id));
2577 }
2578
2579 device->sas_address = get_unaligned_be64(&device->wwid[0]);
2580
2581 new_device_list[num_valid_devices++] = device;
2582 }
2583
2584 pqi_update_device_list(ctrl_info, new_device_list, num_valid_devices);
2585
2586out:
2587 list_for_each_entry_safe(device, next, &new_device_list_head,
2588 new_device_list_entry) {
2589 if (device->keep_device)
2590 continue;
2591 list_del(&device->new_device_list_entry);
2592 pqi_free_device(device);
2593 }
2594
2595 kfree(new_device_list);
2596 kfree(physdev_list);
2597 kfree(logdev_list);
2598 kfree(id_phys);
2599
2600 return rc;
2601}
2602
2603static int pqi_scan_scsi_devices(struct pqi_ctrl_info *ctrl_info)
2604{
2605 int rc;
2606 int mutex_acquired;
2607
2608 if (pqi_ctrl_offline(ctrl_info))
2609 return -ENXIO;
2610
2611 mutex_acquired = mutex_trylock(&ctrl_info->scan_mutex);
2612
2613 if (!mutex_acquired) {
2614 if (pqi_ctrl_scan_blocked(ctrl_info))
2615 return -EBUSY;
2616 pqi_schedule_rescan_worker_delayed(ctrl_info);
2617 return -EINPROGRESS;
2618 }
2619
2620 rc = pqi_update_scsi_devices(ctrl_info);
2621 if (rc && !pqi_ctrl_scan_blocked(ctrl_info))
2622 pqi_schedule_rescan_worker_delayed(ctrl_info);
2623
2624 mutex_unlock(&ctrl_info->scan_mutex);
2625
2626 return rc;
2627}
2628
2629static void pqi_scan_start(struct Scsi_Host *shost)
2630{
2631 struct pqi_ctrl_info *ctrl_info;
2632
2633 ctrl_info = shost_to_hba(shost);
2634
2635 pqi_scan_scsi_devices(ctrl_info);
2636}
2637
2638/* Returns TRUE if scan is finished. */
2639
2640static int pqi_scan_finished(struct Scsi_Host *shost,
2641 unsigned long elapsed_time)
2642{
2643 struct pqi_ctrl_info *ctrl_info;
2644
2645 ctrl_info = shost_priv(shost);
2646
2647 return !mutex_is_locked(&ctrl_info->scan_mutex);
2648}
2649
2650static inline void pqi_set_encryption_info(struct pqi_encryption_info *encryption_info,
2651 struct raid_map *raid_map, u64 first_block)
2652{
2653 u32 volume_blk_size;
2654
2655 /*
2656 * Set the encryption tweak values based on logical block address.
2657 * If the block size is 512, the tweak value is equal to the LBA.
2658 * For other block sizes, tweak value is (LBA * block size) / 512.
2659 */
2660 volume_blk_size = get_unaligned_le32(&raid_map->volume_blk_size);
2661 if (volume_blk_size != 512)
2662 first_block = (first_block * volume_blk_size) / 512;
2663
2664 encryption_info->data_encryption_key_index =
2665 get_unaligned_le16(&raid_map->data_encryption_key_index);
2666 encryption_info->encrypt_tweak_lower = lower_32_bits(first_block);
2667 encryption_info->encrypt_tweak_upper = upper_32_bits(first_block);
2668}
2669
2670/*
2671 * Attempt to perform RAID bypass mapping for a logical volume I/O.
2672 */
2673
2674static bool pqi_aio_raid_level_supported(struct pqi_ctrl_info *ctrl_info,
2675 struct pqi_scsi_dev_raid_map_data *rmd)
2676{
2677 bool is_supported = true;
2678
2679 switch (rmd->raid_level) {
2680 case SA_RAID_0:
2681 break;
2682 case SA_RAID_1:
2683 if (rmd->is_write && (!ctrl_info->enable_r1_writes ||
2684 rmd->data_length > ctrl_info->max_write_raid_1_10_2drive))
2685 is_supported = false;
2686 break;
2687 case SA_RAID_TRIPLE:
2688 if (rmd->is_write && (!ctrl_info->enable_r1_writes ||
2689 rmd->data_length > ctrl_info->max_write_raid_1_10_3drive))
2690 is_supported = false;
2691 break;
2692 case SA_RAID_5:
2693 if (rmd->is_write && (!ctrl_info->enable_r5_writes ||
2694 rmd->data_length > ctrl_info->max_write_raid_5_6))
2695 is_supported = false;
2696 break;
2697 case SA_RAID_6:
2698 if (rmd->is_write && (!ctrl_info->enable_r6_writes ||
2699 rmd->data_length > ctrl_info->max_write_raid_5_6))
2700 is_supported = false;
2701 break;
2702 default:
2703 is_supported = false;
2704 break;
2705 }
2706
2707 return is_supported;
2708}
2709
2710#define PQI_RAID_BYPASS_INELIGIBLE 1
2711
2712static int pqi_get_aio_lba_and_block_count(struct scsi_cmnd *scmd,
2713 struct pqi_scsi_dev_raid_map_data *rmd)
2714{
2715 /* Check for valid opcode, get LBA and block count. */
2716 switch (scmd->cmnd[0]) {
2717 case WRITE_6:
2718 rmd->is_write = true;
2719 fallthrough;
2720 case READ_6:
2721 rmd->first_block = (u64)(((scmd->cmnd[1] & 0x1f) << 16) |
2722 (scmd->cmnd[2] << 8) | scmd->cmnd[3]);
2723 rmd->block_cnt = (u32)scmd->cmnd[4];
2724 if (rmd->block_cnt == 0)
2725 rmd->block_cnt = 256;
2726 break;
2727 case WRITE_10:
2728 rmd->is_write = true;
2729 fallthrough;
2730 case READ_10:
2731 rmd->first_block = (u64)get_unaligned_be32(&scmd->cmnd[2]);
2732 rmd->block_cnt = (u32)get_unaligned_be16(&scmd->cmnd[7]);
2733 break;
2734 case WRITE_12:
2735 rmd->is_write = true;
2736 fallthrough;
2737 case READ_12:
2738 rmd->first_block = (u64)get_unaligned_be32(&scmd->cmnd[2]);
2739 rmd->block_cnt = get_unaligned_be32(&scmd->cmnd[6]);
2740 break;
2741 case WRITE_16:
2742 rmd->is_write = true;
2743 fallthrough;
2744 case READ_16:
2745 rmd->first_block = get_unaligned_be64(&scmd->cmnd[2]);
2746 rmd->block_cnt = get_unaligned_be32(&scmd->cmnd[10]);
2747 break;
2748 default:
2749 /* Process via normal I/O path. */
2750 return PQI_RAID_BYPASS_INELIGIBLE;
2751 }
2752
2753 put_unaligned_le32(scsi_bufflen(scmd), &rmd->data_length);
2754
2755 return 0;
2756}
2757
2758static int pci_get_aio_common_raid_map_values(struct pqi_ctrl_info *ctrl_info,
2759 struct pqi_scsi_dev_raid_map_data *rmd, struct raid_map *raid_map)
2760{
2761#if BITS_PER_LONG == 32
2762 u64 tmpdiv;
2763#endif
2764
2765 rmd->last_block = rmd->first_block + rmd->block_cnt - 1;
2766
2767 /* Check for invalid block or wraparound. */
2768 if (rmd->last_block >=
2769 get_unaligned_le64(&raid_map->volume_blk_cnt) ||
2770 rmd->last_block < rmd->first_block)
2771 return PQI_RAID_BYPASS_INELIGIBLE;
2772
2773 rmd->data_disks_per_row =
2774 get_unaligned_le16(&raid_map->data_disks_per_row);
2775 rmd->strip_size = get_unaligned_le16(&raid_map->strip_size);
2776 rmd->layout_map_count = get_unaligned_le16(&raid_map->layout_map_count);
2777
2778 /* Calculate stripe information for the request. */
2779 rmd->blocks_per_row = rmd->data_disks_per_row * rmd->strip_size;
2780 if (rmd->blocks_per_row == 0) /* Used as a divisor in many calculations */
2781 return PQI_RAID_BYPASS_INELIGIBLE;
2782#if BITS_PER_LONG == 32
2783 tmpdiv = rmd->first_block;
2784 do_div(tmpdiv, rmd->blocks_per_row);
2785 rmd->first_row = tmpdiv;
2786 tmpdiv = rmd->last_block;
2787 do_div(tmpdiv, rmd->blocks_per_row);
2788 rmd->last_row = tmpdiv;
2789 rmd->first_row_offset = (u32)(rmd->first_block - (rmd->first_row * rmd->blocks_per_row));
2790 rmd->last_row_offset = (u32)(rmd->last_block - (rmd->last_row * rmd->blocks_per_row));
2791 tmpdiv = rmd->first_row_offset;
2792 do_div(tmpdiv, rmd->strip_size);
2793 rmd->first_column = tmpdiv;
2794 tmpdiv = rmd->last_row_offset;
2795 do_div(tmpdiv, rmd->strip_size);
2796 rmd->last_column = tmpdiv;
2797#else
2798 rmd->first_row = rmd->first_block / rmd->blocks_per_row;
2799 rmd->last_row = rmd->last_block / rmd->blocks_per_row;
2800 rmd->first_row_offset = (u32)(rmd->first_block -
2801 (rmd->first_row * rmd->blocks_per_row));
2802 rmd->last_row_offset = (u32)(rmd->last_block - (rmd->last_row *
2803 rmd->blocks_per_row));
2804 rmd->first_column = rmd->first_row_offset / rmd->strip_size;
2805 rmd->last_column = rmd->last_row_offset / rmd->strip_size;
2806#endif
2807
2808 /* If this isn't a single row/column then give to the controller. */
2809 if (rmd->first_row != rmd->last_row ||
2810 rmd->first_column != rmd->last_column)
2811 return PQI_RAID_BYPASS_INELIGIBLE;
2812
2813 /* Proceeding with driver mapping. */
2814 rmd->total_disks_per_row = rmd->data_disks_per_row +
2815 get_unaligned_le16(&raid_map->metadata_disks_per_row);
2816 rmd->map_row = ((u32)(rmd->first_row >>
2817 raid_map->parity_rotation_shift)) %
2818 get_unaligned_le16(&raid_map->row_cnt);
2819 rmd->map_index = (rmd->map_row * rmd->total_disks_per_row) +
2820 rmd->first_column;
2821
2822 return 0;
2823}
2824
2825static int pqi_calc_aio_r5_or_r6(struct pqi_scsi_dev_raid_map_data *rmd,
2826 struct raid_map *raid_map)
2827{
2828#if BITS_PER_LONG == 32
2829 u64 tmpdiv;
2830#endif
2831
2832 if (rmd->blocks_per_row == 0) /* Used as a divisor in many calculations */
2833 return PQI_RAID_BYPASS_INELIGIBLE;
2834
2835 /* RAID 50/60 */
2836 /* Verify first and last block are in same RAID group. */
2837 rmd->stripesize = rmd->blocks_per_row * rmd->layout_map_count;
2838#if BITS_PER_LONG == 32
2839 tmpdiv = rmd->first_block;
2840 rmd->first_group = do_div(tmpdiv, rmd->stripesize);
2841 tmpdiv = rmd->first_group;
2842 do_div(tmpdiv, rmd->blocks_per_row);
2843 rmd->first_group = tmpdiv;
2844 tmpdiv = rmd->last_block;
2845 rmd->last_group = do_div(tmpdiv, rmd->stripesize);
2846 tmpdiv = rmd->last_group;
2847 do_div(tmpdiv, rmd->blocks_per_row);
2848 rmd->last_group = tmpdiv;
2849#else
2850 rmd->first_group = (rmd->first_block % rmd->stripesize) / rmd->blocks_per_row;
2851 rmd->last_group = (rmd->last_block % rmd->stripesize) / rmd->blocks_per_row;
2852#endif
2853 if (rmd->first_group != rmd->last_group)
2854 return PQI_RAID_BYPASS_INELIGIBLE;
2855
2856 /* Verify request is in a single row of RAID 5/6. */
2857#if BITS_PER_LONG == 32
2858 tmpdiv = rmd->first_block;
2859 do_div(tmpdiv, rmd->stripesize);
2860 rmd->first_row = tmpdiv;
2861 rmd->r5or6_first_row = tmpdiv;
2862 tmpdiv = rmd->last_block;
2863 do_div(tmpdiv, rmd->stripesize);
2864 rmd->r5or6_last_row = tmpdiv;
2865#else
2866 rmd->first_row = rmd->r5or6_first_row =
2867 rmd->first_block / rmd->stripesize;
2868 rmd->r5or6_last_row = rmd->last_block / rmd->stripesize;
2869#endif
2870 if (rmd->r5or6_first_row != rmd->r5or6_last_row)
2871 return PQI_RAID_BYPASS_INELIGIBLE;
2872
2873 /* Verify request is in a single column. */
2874#if BITS_PER_LONG == 32
2875 tmpdiv = rmd->first_block;
2876 rmd->first_row_offset = do_div(tmpdiv, rmd->stripesize);
2877 tmpdiv = rmd->first_row_offset;
2878 rmd->first_row_offset = (u32)do_div(tmpdiv, rmd->blocks_per_row);
2879 rmd->r5or6_first_row_offset = rmd->first_row_offset;
2880 tmpdiv = rmd->last_block;
2881 rmd->r5or6_last_row_offset = do_div(tmpdiv, rmd->stripesize);
2882 tmpdiv = rmd->r5or6_last_row_offset;
2883 rmd->r5or6_last_row_offset = do_div(tmpdiv, rmd->blocks_per_row);
2884 tmpdiv = rmd->r5or6_first_row_offset;
2885 do_div(tmpdiv, rmd->strip_size);
2886 rmd->first_column = rmd->r5or6_first_column = tmpdiv;
2887 tmpdiv = rmd->r5or6_last_row_offset;
2888 do_div(tmpdiv, rmd->strip_size);
2889 rmd->r5or6_last_column = tmpdiv;
2890#else
2891 rmd->first_row_offset = rmd->r5or6_first_row_offset =
2892 (u32)((rmd->first_block % rmd->stripesize) %
2893 rmd->blocks_per_row);
2894
2895 rmd->r5or6_last_row_offset =
2896 (u32)((rmd->last_block % rmd->stripesize) %
2897 rmd->blocks_per_row);
2898
2899 rmd->first_column =
2900 rmd->r5or6_first_row_offset / rmd->strip_size;
2901 rmd->r5or6_first_column = rmd->first_column;
2902 rmd->r5or6_last_column = rmd->r5or6_last_row_offset / rmd->strip_size;
2903#endif
2904 if (rmd->r5or6_first_column != rmd->r5or6_last_column)
2905 return PQI_RAID_BYPASS_INELIGIBLE;
2906
2907 /* Request is eligible. */
2908 rmd->map_row =
2909 ((u32)(rmd->first_row >> raid_map->parity_rotation_shift)) %
2910 get_unaligned_le16(&raid_map->row_cnt);
2911
2912 rmd->map_index = (rmd->first_group *
2913 (get_unaligned_le16(&raid_map->row_cnt) *
2914 rmd->total_disks_per_row)) +
2915 (rmd->map_row * rmd->total_disks_per_row) + rmd->first_column;
2916
2917 if (rmd->is_write) {
2918 u32 index;
2919
2920 /*
2921 * p_parity_it_nexus and q_parity_it_nexus are pointers to the
2922 * parity entries inside the device's raid_map.
2923 *
2924 * A device's RAID map is bounded by: number of RAID disks squared.
2925 *
2926 * The devices RAID map size is checked during device
2927 * initialization.
2928 */
2929 index = DIV_ROUND_UP(rmd->map_index + 1, rmd->total_disks_per_row);
2930 index *= rmd->total_disks_per_row;
2931 index -= get_unaligned_le16(&raid_map->metadata_disks_per_row);
2932
2933 rmd->p_parity_it_nexus = raid_map->disk_data[index].aio_handle;
2934 if (rmd->raid_level == SA_RAID_6) {
2935 rmd->q_parity_it_nexus = raid_map->disk_data[index + 1].aio_handle;
2936 rmd->xor_mult = raid_map->disk_data[rmd->map_index].xor_mult[1];
2937 }
2938#if BITS_PER_LONG == 32
2939 tmpdiv = rmd->first_block;
2940 do_div(tmpdiv, rmd->blocks_per_row);
2941 rmd->row = tmpdiv;
2942#else
2943 rmd->row = rmd->first_block / rmd->blocks_per_row;
2944#endif
2945 }
2946
2947 return 0;
2948}
2949
2950static void pqi_set_aio_cdb(struct pqi_scsi_dev_raid_map_data *rmd)
2951{
2952 /* Build the new CDB for the physical disk I/O. */
2953 if (rmd->disk_block > 0xffffffff) {
2954 rmd->cdb[0] = rmd->is_write ? WRITE_16 : READ_16;
2955 rmd->cdb[1] = 0;
2956 put_unaligned_be64(rmd->disk_block, &rmd->cdb[2]);
2957 put_unaligned_be32(rmd->disk_block_cnt, &rmd->cdb[10]);
2958 rmd->cdb[14] = 0;
2959 rmd->cdb[15] = 0;
2960 rmd->cdb_length = 16;
2961 } else {
2962 rmd->cdb[0] = rmd->is_write ? WRITE_10 : READ_10;
2963 rmd->cdb[1] = 0;
2964 put_unaligned_be32((u32)rmd->disk_block, &rmd->cdb[2]);
2965 rmd->cdb[6] = 0;
2966 put_unaligned_be16((u16)rmd->disk_block_cnt, &rmd->cdb[7]);
2967 rmd->cdb[9] = 0;
2968 rmd->cdb_length = 10;
2969 }
2970}
2971
2972static void pqi_calc_aio_r1_nexus(struct raid_map *raid_map,
2973 struct pqi_scsi_dev_raid_map_data *rmd)
2974{
2975 u32 index;
2976 u32 group;
2977
2978 group = rmd->map_index / rmd->data_disks_per_row;
2979
2980 index = rmd->map_index - (group * rmd->data_disks_per_row);
2981 rmd->it_nexus[0] = raid_map->disk_data[index].aio_handle;
2982 index += rmd->data_disks_per_row;
2983 rmd->it_nexus[1] = raid_map->disk_data[index].aio_handle;
2984 if (rmd->layout_map_count > 2) {
2985 index += rmd->data_disks_per_row;
2986 rmd->it_nexus[2] = raid_map->disk_data[index].aio_handle;
2987 }
2988
2989 rmd->num_it_nexus_entries = rmd->layout_map_count;
2990}
2991
2992static int pqi_raid_bypass_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,
2993 struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
2994 struct pqi_queue_group *queue_group)
2995{
2996 int rc;
2997 struct raid_map *raid_map;
2998 u32 group;
2999 u32 next_bypass_group;
3000 struct pqi_encryption_info *encryption_info_ptr;
3001 struct pqi_encryption_info encryption_info;
3002 struct pqi_scsi_dev_raid_map_data rmd = { 0 };
3003
3004 rc = pqi_get_aio_lba_and_block_count(scmd, &rmd);
3005 if (rc)
3006 return PQI_RAID_BYPASS_INELIGIBLE;
3007
3008 rmd.raid_level = device->raid_level;
3009
3010 if (!pqi_aio_raid_level_supported(ctrl_info, &rmd))
3011 return PQI_RAID_BYPASS_INELIGIBLE;
3012
3013 if (unlikely(rmd.block_cnt == 0))
3014 return PQI_RAID_BYPASS_INELIGIBLE;
3015
3016 raid_map = device->raid_map;
3017
3018 rc = pci_get_aio_common_raid_map_values(ctrl_info, &rmd, raid_map);
3019 if (rc)
3020 return PQI_RAID_BYPASS_INELIGIBLE;
3021
3022 if (device->raid_level == SA_RAID_1 ||
3023 device->raid_level == SA_RAID_TRIPLE) {
3024 if (rmd.is_write) {
3025 pqi_calc_aio_r1_nexus(raid_map, &rmd);
3026 } else {
3027 group = device->next_bypass_group[rmd.map_index];
3028 next_bypass_group = group + 1;
3029 if (next_bypass_group >= rmd.layout_map_count)
3030 next_bypass_group = 0;
3031 device->next_bypass_group[rmd.map_index] = next_bypass_group;
3032 rmd.map_index += group * rmd.data_disks_per_row;
3033 }
3034 } else if ((device->raid_level == SA_RAID_5 ||
3035 device->raid_level == SA_RAID_6) &&
3036 (rmd.layout_map_count > 1 || rmd.is_write)) {
3037 rc = pqi_calc_aio_r5_or_r6(&rmd, raid_map);
3038 if (rc)
3039 return PQI_RAID_BYPASS_INELIGIBLE;
3040 }
3041
3042 if (unlikely(rmd.map_index >= RAID_MAP_MAX_ENTRIES))
3043 return PQI_RAID_BYPASS_INELIGIBLE;
3044
3045 rmd.aio_handle = raid_map->disk_data[rmd.map_index].aio_handle;
3046 rmd.disk_block = get_unaligned_le64(&raid_map->disk_starting_blk) +
3047 rmd.first_row * rmd.strip_size +
3048 (rmd.first_row_offset - rmd.first_column * rmd.strip_size);
3049 rmd.disk_block_cnt = rmd.block_cnt;
3050
3051 /* Handle differing logical/physical block sizes. */
3052 if (raid_map->phys_blk_shift) {
3053 rmd.disk_block <<= raid_map->phys_blk_shift;
3054 rmd.disk_block_cnt <<= raid_map->phys_blk_shift;
3055 }
3056
3057 if (unlikely(rmd.disk_block_cnt > 0xffff))
3058 return PQI_RAID_BYPASS_INELIGIBLE;
3059
3060 pqi_set_aio_cdb(&rmd);
3061
3062 if (get_unaligned_le16(&raid_map->flags) & RAID_MAP_ENCRYPTION_ENABLED) {
3063 if (rmd.data_length > device->max_transfer_encrypted)
3064 return PQI_RAID_BYPASS_INELIGIBLE;
3065 pqi_set_encryption_info(&encryption_info, raid_map, rmd.first_block);
3066 encryption_info_ptr = &encryption_info;
3067 } else {
3068 encryption_info_ptr = NULL;
3069 }
3070
3071 if (rmd.is_write) {
3072 switch (device->raid_level) {
3073 case SA_RAID_1:
3074 case SA_RAID_TRIPLE:
3075 return pqi_aio_submit_r1_write_io(ctrl_info, scmd, queue_group,
3076 encryption_info_ptr, device, &rmd);
3077 case SA_RAID_5:
3078 case SA_RAID_6:
3079 return pqi_aio_submit_r56_write_io(ctrl_info, scmd, queue_group,
3080 encryption_info_ptr, device, &rmd);
3081 }
3082 }
3083
3084 return pqi_aio_submit_io(ctrl_info, scmd, rmd.aio_handle,
3085 rmd.cdb, rmd.cdb_length, queue_group,
3086 encryption_info_ptr, true, false);
3087}
3088
3089#define PQI_STATUS_IDLE 0x0
3090
3091#define PQI_CREATE_ADMIN_QUEUE_PAIR 1
3092#define PQI_DELETE_ADMIN_QUEUE_PAIR 2
3093
3094#define PQI_DEVICE_STATE_POWER_ON_AND_RESET 0x0
3095#define PQI_DEVICE_STATE_STATUS_AVAILABLE 0x1
3096#define PQI_DEVICE_STATE_ALL_REGISTERS_READY 0x2
3097#define PQI_DEVICE_STATE_ADMIN_QUEUE_PAIR_READY 0x3
3098#define PQI_DEVICE_STATE_ERROR 0x4
3099
3100#define PQI_MODE_READY_TIMEOUT_SECS 30
3101#define PQI_MODE_READY_POLL_INTERVAL_MSECS 1
3102
3103static int pqi_wait_for_pqi_mode_ready(struct pqi_ctrl_info *ctrl_info)
3104{
3105 struct pqi_device_registers __iomem *pqi_registers;
3106 unsigned long timeout;
3107 u64 signature;
3108 u8 status;
3109
3110 pqi_registers = ctrl_info->pqi_registers;
3111 timeout = (PQI_MODE_READY_TIMEOUT_SECS * HZ) + jiffies;
3112
3113 while (1) {
3114 signature = readq(&pqi_registers->signature);
3115 if (memcmp(&signature, PQI_DEVICE_SIGNATURE,
3116 sizeof(signature)) == 0)
3117 break;
3118 if (time_after(jiffies, timeout)) {
3119 dev_err(&ctrl_info->pci_dev->dev,
3120 "timed out waiting for PQI signature\n");
3121 return -ETIMEDOUT;
3122 }
3123 msleep(PQI_MODE_READY_POLL_INTERVAL_MSECS);
3124 }
3125
3126 while (1) {
3127 status = readb(&pqi_registers->function_and_status_code);
3128 if (status == PQI_STATUS_IDLE)
3129 break;
3130 if (time_after(jiffies, timeout)) {
3131 dev_err(&ctrl_info->pci_dev->dev,
3132 "timed out waiting for PQI IDLE\n");
3133 return -ETIMEDOUT;
3134 }
3135 msleep(PQI_MODE_READY_POLL_INTERVAL_MSECS);
3136 }
3137
3138 while (1) {
3139 if (readl(&pqi_registers->device_status) ==
3140 PQI_DEVICE_STATE_ALL_REGISTERS_READY)
3141 break;
3142 if (time_after(jiffies, timeout)) {
3143 dev_err(&ctrl_info->pci_dev->dev,
3144 "timed out waiting for PQI all registers ready\n");
3145 return -ETIMEDOUT;
3146 }
3147 msleep(PQI_MODE_READY_POLL_INTERVAL_MSECS);
3148 }
3149
3150 return 0;
3151}
3152
3153static inline void pqi_aio_path_disabled(struct pqi_io_request *io_request)
3154{
3155 struct pqi_scsi_dev *device;
3156
3157 device = io_request->scmd->device->hostdata;
3158 device->raid_bypass_enabled = false;
3159 device->aio_enabled = false;
3160}
3161
3162static inline void pqi_take_device_offline(struct scsi_device *sdev, char *path)
3163{
3164 struct pqi_ctrl_info *ctrl_info;
3165 struct pqi_scsi_dev *device;
3166
3167 device = sdev->hostdata;
3168 if (device->device_offline)
3169 return;
3170
3171 device->device_offline = true;
3172 ctrl_info = shost_to_hba(sdev->host);
3173 pqi_schedule_rescan_worker(ctrl_info);
3174 dev_err(&ctrl_info->pci_dev->dev, "re-scanning %s scsi %d:%d:%d:%d\n",
3175 path, ctrl_info->scsi_host->host_no, device->bus,
3176 device->target, device->lun);
3177}
3178
3179static void pqi_process_raid_io_error(struct pqi_io_request *io_request)
3180{
3181 u8 scsi_status;
3182 u8 host_byte;
3183 struct scsi_cmnd *scmd;
3184 struct pqi_raid_error_info *error_info;
3185 size_t sense_data_length;
3186 int residual_count;
3187 int xfer_count;
3188 struct scsi_sense_hdr sshdr;
3189
3190 scmd = io_request->scmd;
3191 if (!scmd)
3192 return;
3193
3194 error_info = io_request->error_info;
3195 scsi_status = error_info->status;
3196 host_byte = DID_OK;
3197
3198 switch (error_info->data_out_result) {
3199 case PQI_DATA_IN_OUT_GOOD:
3200 break;
3201 case PQI_DATA_IN_OUT_UNDERFLOW:
3202 xfer_count =
3203 get_unaligned_le32(&error_info->data_out_transferred);
3204 residual_count = scsi_bufflen(scmd) - xfer_count;
3205 scsi_set_resid(scmd, residual_count);
3206 if (xfer_count < scmd->underflow)
3207 host_byte = DID_SOFT_ERROR;
3208 break;
3209 case PQI_DATA_IN_OUT_UNSOLICITED_ABORT:
3210 case PQI_DATA_IN_OUT_ABORTED:
3211 host_byte = DID_ABORT;
3212 break;
3213 case PQI_DATA_IN_OUT_TIMEOUT:
3214 host_byte = DID_TIME_OUT;
3215 break;
3216 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW:
3217 case PQI_DATA_IN_OUT_PROTOCOL_ERROR:
3218 case PQI_DATA_IN_OUT_BUFFER_ERROR:
3219 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_DESCRIPTOR_AREA:
3220 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_BRIDGE:
3221 case PQI_DATA_IN_OUT_ERROR:
3222 case PQI_DATA_IN_OUT_HARDWARE_ERROR:
3223 case PQI_DATA_IN_OUT_PCIE_FABRIC_ERROR:
3224 case PQI_DATA_IN_OUT_PCIE_COMPLETION_TIMEOUT:
3225 case PQI_DATA_IN_OUT_PCIE_COMPLETER_ABORT_RECEIVED:
3226 case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST_RECEIVED:
3227 case PQI_DATA_IN_OUT_PCIE_ECRC_CHECK_FAILED:
3228 case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST:
3229 case PQI_DATA_IN_OUT_PCIE_ACS_VIOLATION:
3230 case PQI_DATA_IN_OUT_PCIE_TLP_PREFIX_BLOCKED:
3231 case PQI_DATA_IN_OUT_PCIE_POISONED_MEMORY_READ:
3232 default:
3233 host_byte = DID_ERROR;
3234 break;
3235 }
3236
3237 sense_data_length = get_unaligned_le16(&error_info->sense_data_length);
3238 if (sense_data_length == 0)
3239 sense_data_length =
3240 get_unaligned_le16(&error_info->response_data_length);
3241 if (sense_data_length) {
3242 if (sense_data_length > sizeof(error_info->data))
3243 sense_data_length = sizeof(error_info->data);
3244
3245 if (scsi_status == SAM_STAT_CHECK_CONDITION &&
3246 scsi_normalize_sense(error_info->data,
3247 sense_data_length, &sshdr) &&
3248 sshdr.sense_key == HARDWARE_ERROR &&
3249 sshdr.asc == 0x3e) {
3250 struct pqi_ctrl_info *ctrl_info = shost_to_hba(scmd->device->host);
3251 struct pqi_scsi_dev *device = scmd->device->hostdata;
3252
3253 switch (sshdr.ascq) {
3254 case 0x1: /* LOGICAL UNIT FAILURE */
3255 if (printk_ratelimit())
3256 scmd_printk(KERN_ERR, scmd, "received 'logical unit failure' from controller for scsi %d:%d:%d:%d\n",
3257 ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun);
3258 pqi_take_device_offline(scmd->device, "RAID");
3259 host_byte = DID_NO_CONNECT;
3260 break;
3261
3262 default: /* See http://www.t10.org/lists/asc-num.htm#ASC_3E */
3263 if (printk_ratelimit())
3264 scmd_printk(KERN_ERR, scmd, "received unhandled error %d from controller for scsi %d:%d:%d:%d\n",
3265 sshdr.ascq, ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun);
3266 break;
3267 }
3268 }
3269
3270 if (sense_data_length > SCSI_SENSE_BUFFERSIZE)
3271 sense_data_length = SCSI_SENSE_BUFFERSIZE;
3272 memcpy(scmd->sense_buffer, error_info->data,
3273 sense_data_length);
3274 }
3275
3276 if (pqi_cmd_priv(scmd)->this_residual &&
3277 !pqi_is_logical_device(scmd->device->hostdata) &&
3278 scsi_status == SAM_STAT_CHECK_CONDITION &&
3279 host_byte == DID_OK &&
3280 sense_data_length &&
3281 scsi_normalize_sense(error_info->data, sense_data_length, &sshdr) &&
3282 sshdr.sense_key == ILLEGAL_REQUEST &&
3283 sshdr.asc == 0x26 &&
3284 sshdr.ascq == 0x0) {
3285 host_byte = DID_NO_CONNECT;
3286 pqi_take_device_offline(scmd->device, "AIO");
3287 scsi_build_sense_buffer(0, scmd->sense_buffer, HARDWARE_ERROR, 0x3e, 0x1);
3288 }
3289
3290 scmd->result = scsi_status;
3291 set_host_byte(scmd, host_byte);
3292}
3293
3294static void pqi_process_aio_io_error(struct pqi_io_request *io_request)
3295{
3296 u8 scsi_status;
3297 u8 host_byte;
3298 struct scsi_cmnd *scmd;
3299 struct pqi_aio_error_info *error_info;
3300 size_t sense_data_length;
3301 int residual_count;
3302 int xfer_count;
3303 bool device_offline;
3304
3305 scmd = io_request->scmd;
3306 error_info = io_request->error_info;
3307 host_byte = DID_OK;
3308 sense_data_length = 0;
3309 device_offline = false;
3310
3311 switch (error_info->service_response) {
3312 case PQI_AIO_SERV_RESPONSE_COMPLETE:
3313 scsi_status = error_info->status;
3314 break;
3315 case PQI_AIO_SERV_RESPONSE_FAILURE:
3316 switch (error_info->status) {
3317 case PQI_AIO_STATUS_IO_ABORTED:
3318 scsi_status = SAM_STAT_TASK_ABORTED;
3319 break;
3320 case PQI_AIO_STATUS_UNDERRUN:
3321 scsi_status = SAM_STAT_GOOD;
3322 residual_count = get_unaligned_le32(
3323 &error_info->residual_count);
3324 scsi_set_resid(scmd, residual_count);
3325 xfer_count = scsi_bufflen(scmd) - residual_count;
3326 if (xfer_count < scmd->underflow)
3327 host_byte = DID_SOFT_ERROR;
3328 break;
3329 case PQI_AIO_STATUS_OVERRUN:
3330 scsi_status = SAM_STAT_GOOD;
3331 break;
3332 case PQI_AIO_STATUS_AIO_PATH_DISABLED:
3333 pqi_aio_path_disabled(io_request);
3334 scsi_status = SAM_STAT_GOOD;
3335 io_request->status = -EAGAIN;
3336 break;
3337 case PQI_AIO_STATUS_NO_PATH_TO_DEVICE:
3338 case PQI_AIO_STATUS_INVALID_DEVICE:
3339 if (!io_request->raid_bypass) {
3340 device_offline = true;
3341 pqi_take_device_offline(scmd->device, "AIO");
3342 host_byte = DID_NO_CONNECT;
3343 }
3344 scsi_status = SAM_STAT_CHECK_CONDITION;
3345 break;
3346 case PQI_AIO_STATUS_IO_ERROR:
3347 default:
3348 scsi_status = SAM_STAT_CHECK_CONDITION;
3349 break;
3350 }
3351 break;
3352 case PQI_AIO_SERV_RESPONSE_TMF_COMPLETE:
3353 case PQI_AIO_SERV_RESPONSE_TMF_SUCCEEDED:
3354 scsi_status = SAM_STAT_GOOD;
3355 break;
3356 case PQI_AIO_SERV_RESPONSE_TMF_REJECTED:
3357 case PQI_AIO_SERV_RESPONSE_TMF_INCORRECT_LUN:
3358 default:
3359 scsi_status = SAM_STAT_CHECK_CONDITION;
3360 break;
3361 }
3362
3363 if (error_info->data_present) {
3364 sense_data_length =
3365 get_unaligned_le16(&error_info->data_length);
3366 if (sense_data_length) {
3367 if (sense_data_length > sizeof(error_info->data))
3368 sense_data_length = sizeof(error_info->data);
3369 if (sense_data_length > SCSI_SENSE_BUFFERSIZE)
3370 sense_data_length = SCSI_SENSE_BUFFERSIZE;
3371 memcpy(scmd->sense_buffer, error_info->data,
3372 sense_data_length);
3373 }
3374 }
3375
3376 if (device_offline && sense_data_length == 0)
3377 scsi_build_sense(scmd, 0, HARDWARE_ERROR, 0x3e, 0x1);
3378
3379 scmd->result = scsi_status;
3380 set_host_byte(scmd, host_byte);
3381}
3382
3383static void pqi_process_io_error(unsigned int iu_type,
3384 struct pqi_io_request *io_request)
3385{
3386 switch (iu_type) {
3387 case PQI_RESPONSE_IU_RAID_PATH_IO_ERROR:
3388 pqi_process_raid_io_error(io_request);
3389 break;
3390 case PQI_RESPONSE_IU_AIO_PATH_IO_ERROR:
3391 pqi_process_aio_io_error(io_request);
3392 break;
3393 }
3394}
3395
3396static int pqi_interpret_task_management_response(struct pqi_ctrl_info *ctrl_info,
3397 struct pqi_task_management_response *response)
3398{
3399 int rc;
3400
3401 switch (response->response_code) {
3402 case SOP_TMF_COMPLETE:
3403 case SOP_TMF_FUNCTION_SUCCEEDED:
3404 rc = 0;
3405 break;
3406 case SOP_TMF_REJECTED:
3407 rc = -EAGAIN;
3408 break;
3409 case SOP_TMF_INCORRECT_LOGICAL_UNIT:
3410 rc = -ENODEV;
3411 break;
3412 default:
3413 rc = -EIO;
3414 break;
3415 }
3416
3417 if (rc)
3418 dev_err(&ctrl_info->pci_dev->dev,
3419 "Task Management Function error: %d (response code: %u)\n", rc, response->response_code);
3420
3421 return rc;
3422}
3423
3424static inline void pqi_invalid_response(struct pqi_ctrl_info *ctrl_info,
3425 enum pqi_ctrl_shutdown_reason ctrl_shutdown_reason)
3426{
3427 pqi_take_ctrl_offline(ctrl_info, ctrl_shutdown_reason);
3428}
3429
3430static int pqi_process_io_intr(struct pqi_ctrl_info *ctrl_info, struct pqi_queue_group *queue_group)
3431{
3432 int num_responses;
3433 pqi_index_t oq_pi;
3434 pqi_index_t oq_ci;
3435 struct pqi_io_request *io_request;
3436 struct pqi_io_response *response;
3437 u16 request_id;
3438
3439 num_responses = 0;
3440 oq_ci = queue_group->oq_ci_copy;
3441
3442 while (1) {
3443 oq_pi = readl(queue_group->oq_pi);
3444 if (oq_pi >= ctrl_info->num_elements_per_oq) {
3445 pqi_invalid_response(ctrl_info, PQI_IO_PI_OUT_OF_RANGE);
3446 dev_err(&ctrl_info->pci_dev->dev,
3447 "I/O interrupt: producer index (%u) out of range (0-%u): consumer index: %u\n",
3448 oq_pi, ctrl_info->num_elements_per_oq - 1, oq_ci);
3449 return -1;
3450 }
3451 if (oq_pi == oq_ci)
3452 break;
3453
3454 num_responses++;
3455 response = queue_group->oq_element_array +
3456 (oq_ci * PQI_OPERATIONAL_OQ_ELEMENT_LENGTH);
3457
3458 request_id = get_unaligned_le16(&response->request_id);
3459 if (request_id >= ctrl_info->max_io_slots) {
3460 pqi_invalid_response(ctrl_info, PQI_INVALID_REQ_ID);
3461 dev_err(&ctrl_info->pci_dev->dev,
3462 "request ID in response (%u) out of range (0-%u): producer index: %u consumer index: %u\n",
3463 request_id, ctrl_info->max_io_slots - 1, oq_pi, oq_ci);
3464 return -1;
3465 }
3466
3467 io_request = &ctrl_info->io_request_pool[request_id];
3468 if (atomic_read(&io_request->refcount) == 0) {
3469 pqi_invalid_response(ctrl_info, PQI_UNMATCHED_REQ_ID);
3470 dev_err(&ctrl_info->pci_dev->dev,
3471 "request ID in response (%u) does not match an outstanding I/O request: producer index: %u consumer index: %u\n",
3472 request_id, oq_pi, oq_ci);
3473 return -1;
3474 }
3475
3476 switch (response->header.iu_type) {
3477 case PQI_RESPONSE_IU_RAID_PATH_IO_SUCCESS:
3478 case PQI_RESPONSE_IU_AIO_PATH_IO_SUCCESS:
3479 if (io_request->scmd)
3480 io_request->scmd->result = 0;
3481 fallthrough;
3482 case PQI_RESPONSE_IU_GENERAL_MANAGEMENT:
3483 break;
3484 case PQI_RESPONSE_IU_VENDOR_GENERAL:
3485 io_request->status =
3486 get_unaligned_le16(
3487 &((struct pqi_vendor_general_response *)response)->status);
3488 break;
3489 case PQI_RESPONSE_IU_TASK_MANAGEMENT:
3490 io_request->status = pqi_interpret_task_management_response(ctrl_info,
3491 (void *)response);
3492 break;
3493 case PQI_RESPONSE_IU_AIO_PATH_DISABLED:
3494 pqi_aio_path_disabled(io_request);
3495 io_request->status = -EAGAIN;
3496 break;
3497 case PQI_RESPONSE_IU_RAID_PATH_IO_ERROR:
3498 case PQI_RESPONSE_IU_AIO_PATH_IO_ERROR:
3499 io_request->error_info = ctrl_info->error_buffer +
3500 (get_unaligned_le16(&response->error_index) *
3501 PQI_ERROR_BUFFER_ELEMENT_LENGTH);
3502 pqi_process_io_error(response->header.iu_type, io_request);
3503 break;
3504 default:
3505 pqi_invalid_response(ctrl_info, PQI_UNEXPECTED_IU_TYPE);
3506 dev_err(&ctrl_info->pci_dev->dev,
3507 "unexpected IU type: 0x%x: producer index: %u consumer index: %u\n",
3508 response->header.iu_type, oq_pi, oq_ci);
3509 return -1;
3510 }
3511
3512 io_request->io_complete_callback(io_request, io_request->context);
3513
3514 /*
3515 * Note that the I/O request structure CANNOT BE TOUCHED after
3516 * returning from the I/O completion callback!
3517 */
3518 oq_ci = (oq_ci + 1) % ctrl_info->num_elements_per_oq;
3519 }
3520
3521 if (num_responses) {
3522 queue_group->oq_ci_copy = oq_ci;
3523 writel(oq_ci, queue_group->oq_ci);
3524 }
3525
3526 return num_responses;
3527}
3528
3529static inline unsigned int pqi_num_elements_free(unsigned int pi,
3530 unsigned int ci, unsigned int elements_in_queue)
3531{
3532 unsigned int num_elements_used;
3533
3534 if (pi >= ci)
3535 num_elements_used = pi - ci;
3536 else
3537 num_elements_used = elements_in_queue - ci + pi;
3538
3539 return elements_in_queue - num_elements_used - 1;
3540}
3541
3542static void pqi_send_event_ack(struct pqi_ctrl_info *ctrl_info,
3543 struct pqi_event_acknowledge_request *iu, size_t iu_length)
3544{
3545 pqi_index_t iq_pi;
3546 pqi_index_t iq_ci;
3547 unsigned long flags;
3548 void *next_element;
3549 struct pqi_queue_group *queue_group;
3550
3551 queue_group = &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP];
3552 put_unaligned_le16(queue_group->oq_id, &iu->header.response_queue_id);
3553
3554 while (1) {
3555 spin_lock_irqsave(&queue_group->submit_lock[RAID_PATH], flags);
3556
3557 iq_pi = queue_group->iq_pi_copy[RAID_PATH];
3558 iq_ci = readl(queue_group->iq_ci[RAID_PATH]);
3559
3560 if (pqi_num_elements_free(iq_pi, iq_ci,
3561 ctrl_info->num_elements_per_iq))
3562 break;
3563
3564 spin_unlock_irqrestore(
3565 &queue_group->submit_lock[RAID_PATH], flags);
3566
3567 if (pqi_ctrl_offline(ctrl_info))
3568 return;
3569 }
3570
3571 next_element = queue_group->iq_element_array[RAID_PATH] +
3572 (iq_pi * PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
3573
3574 memcpy(next_element, iu, iu_length);
3575
3576 iq_pi = (iq_pi + 1) % ctrl_info->num_elements_per_iq;
3577 queue_group->iq_pi_copy[RAID_PATH] = iq_pi;
3578
3579 /*
3580 * This write notifies the controller that an IU is available to be
3581 * processed.
3582 */
3583 writel(iq_pi, queue_group->iq_pi[RAID_PATH]);
3584
3585 spin_unlock_irqrestore(&queue_group->submit_lock[RAID_PATH], flags);
3586}
3587
3588static void pqi_acknowledge_event(struct pqi_ctrl_info *ctrl_info,
3589 struct pqi_event *event)
3590{
3591 struct pqi_event_acknowledge_request request;
3592
3593 memset(&request, 0, sizeof(request));
3594
3595 request.header.iu_type = PQI_REQUEST_IU_ACKNOWLEDGE_VENDOR_EVENT;
3596 put_unaligned_le16(sizeof(request) - PQI_REQUEST_HEADER_LENGTH,
3597 &request.header.iu_length);
3598 request.event_type = event->event_type;
3599 put_unaligned_le16(event->event_id, &request.event_id);
3600 put_unaligned_le32(event->additional_event_id, &request.additional_event_id);
3601
3602 pqi_send_event_ack(ctrl_info, &request, sizeof(request));
3603}
3604
3605#define PQI_SOFT_RESET_STATUS_TIMEOUT_SECS 30
3606#define PQI_SOFT_RESET_STATUS_POLL_INTERVAL_SECS 1
3607
3608static enum pqi_soft_reset_status pqi_poll_for_soft_reset_status(
3609 struct pqi_ctrl_info *ctrl_info)
3610{
3611 u8 status;
3612 unsigned long timeout;
3613
3614 timeout = (PQI_SOFT_RESET_STATUS_TIMEOUT_SECS * HZ) + jiffies;
3615
3616 while (1) {
3617 status = pqi_read_soft_reset_status(ctrl_info);
3618 if (status & PQI_SOFT_RESET_INITIATE)
3619 return RESET_INITIATE_DRIVER;
3620
3621 if (status & PQI_SOFT_RESET_ABORT)
3622 return RESET_ABORT;
3623
3624 if (!sis_is_firmware_running(ctrl_info))
3625 return RESET_NORESPONSE;
3626
3627 if (time_after(jiffies, timeout)) {
3628 dev_warn(&ctrl_info->pci_dev->dev,
3629 "timed out waiting for soft reset status\n");
3630 return RESET_TIMEDOUT;
3631 }
3632
3633 ssleep(PQI_SOFT_RESET_STATUS_POLL_INTERVAL_SECS);
3634 }
3635}
3636
3637static void pqi_process_soft_reset(struct pqi_ctrl_info *ctrl_info)
3638{
3639 int rc;
3640 unsigned int delay_secs;
3641 enum pqi_soft_reset_status reset_status;
3642
3643 if (ctrl_info->soft_reset_handshake_supported)
3644 reset_status = pqi_poll_for_soft_reset_status(ctrl_info);
3645 else
3646 reset_status = RESET_INITIATE_FIRMWARE;
3647
3648 delay_secs = PQI_POST_RESET_DELAY_SECS;
3649
3650 switch (reset_status) {
3651 case RESET_TIMEDOUT:
3652 delay_secs = PQI_POST_OFA_RESET_DELAY_UPON_TIMEOUT_SECS;
3653 fallthrough;
3654 case RESET_INITIATE_DRIVER:
3655 dev_info(&ctrl_info->pci_dev->dev,
3656 "Online Firmware Activation: resetting controller\n");
3657 sis_soft_reset(ctrl_info);
3658 fallthrough;
3659 case RESET_INITIATE_FIRMWARE:
3660 ctrl_info->pqi_mode_enabled = false;
3661 pqi_save_ctrl_mode(ctrl_info, SIS_MODE);
3662 rc = pqi_ofa_ctrl_restart(ctrl_info, delay_secs);
3663 pqi_host_free_buffer(ctrl_info, &ctrl_info->ofa_memory);
3664 pqi_ctrl_ofa_done(ctrl_info);
3665 dev_info(&ctrl_info->pci_dev->dev,
3666 "Online Firmware Activation: %s\n",
3667 rc == 0 ? "SUCCESS" : "FAILED");
3668 break;
3669 case RESET_ABORT:
3670 dev_info(&ctrl_info->pci_dev->dev,
3671 "Online Firmware Activation ABORTED\n");
3672 if (ctrl_info->soft_reset_handshake_supported)
3673 pqi_clear_soft_reset_status(ctrl_info);
3674 pqi_host_free_buffer(ctrl_info, &ctrl_info->ofa_memory);
3675 pqi_ctrl_ofa_done(ctrl_info);
3676 pqi_ofa_ctrl_unquiesce(ctrl_info);
3677 break;
3678 case RESET_NORESPONSE:
3679 fallthrough;
3680 default:
3681 dev_err(&ctrl_info->pci_dev->dev,
3682 "unexpected Online Firmware Activation reset status: 0x%x\n",
3683 reset_status);
3684 pqi_host_free_buffer(ctrl_info, &ctrl_info->ofa_memory);
3685 pqi_ctrl_ofa_done(ctrl_info);
3686 pqi_ofa_ctrl_unquiesce(ctrl_info);
3687 pqi_take_ctrl_offline(ctrl_info, PQI_OFA_RESPONSE_TIMEOUT);
3688 break;
3689 }
3690}
3691
3692static void pqi_ofa_memory_alloc_worker(struct work_struct *work)
3693{
3694 struct pqi_ctrl_info *ctrl_info;
3695
3696 ctrl_info = container_of(work, struct pqi_ctrl_info, ofa_memory_alloc_work);
3697
3698 pqi_ctrl_ofa_start(ctrl_info);
3699 pqi_host_setup_buffer(ctrl_info, &ctrl_info->ofa_memory, ctrl_info->ofa_bytes_requested, ctrl_info->ofa_bytes_requested);
3700 pqi_host_memory_update(ctrl_info, &ctrl_info->ofa_memory, PQI_VENDOR_GENERAL_OFA_MEMORY_UPDATE);
3701}
3702
3703static void pqi_ofa_quiesce_worker(struct work_struct *work)
3704{
3705 struct pqi_ctrl_info *ctrl_info;
3706 struct pqi_event *event;
3707
3708 ctrl_info = container_of(work, struct pqi_ctrl_info, ofa_quiesce_work);
3709
3710 event = &ctrl_info->events[pqi_event_type_to_event_index(PQI_EVENT_TYPE_OFA)];
3711
3712 pqi_ofa_ctrl_quiesce(ctrl_info);
3713 pqi_acknowledge_event(ctrl_info, event);
3714 pqi_process_soft_reset(ctrl_info);
3715}
3716
3717static bool pqi_ofa_process_event(struct pqi_ctrl_info *ctrl_info,
3718 struct pqi_event *event)
3719{
3720 bool ack_event;
3721
3722 ack_event = true;
3723
3724 switch (event->event_id) {
3725 case PQI_EVENT_OFA_MEMORY_ALLOCATION:
3726 dev_info(&ctrl_info->pci_dev->dev,
3727 "received Online Firmware Activation memory allocation request\n");
3728 schedule_work(&ctrl_info->ofa_memory_alloc_work);
3729 break;
3730 case PQI_EVENT_OFA_QUIESCE:
3731 dev_info(&ctrl_info->pci_dev->dev,
3732 "received Online Firmware Activation quiesce request\n");
3733 schedule_work(&ctrl_info->ofa_quiesce_work);
3734 ack_event = false;
3735 break;
3736 case PQI_EVENT_OFA_CANCELED:
3737 dev_info(&ctrl_info->pci_dev->dev,
3738 "received Online Firmware Activation cancel request: reason: %u\n",
3739 ctrl_info->ofa_cancel_reason);
3740 pqi_host_free_buffer(ctrl_info, &ctrl_info->ofa_memory);
3741 pqi_ctrl_ofa_done(ctrl_info);
3742 break;
3743 default:
3744 dev_err(&ctrl_info->pci_dev->dev,
3745 "received unknown Online Firmware Activation request: event ID: %u\n",
3746 event->event_id);
3747 break;
3748 }
3749
3750 return ack_event;
3751}
3752
3753static void pqi_mark_volumes_for_rescan(struct pqi_ctrl_info *ctrl_info)
3754{
3755 unsigned long flags;
3756 struct pqi_scsi_dev *device;
3757
3758 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
3759
3760 list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry) {
3761 if (pqi_is_logical_device(device) && device->devtype == TYPE_DISK)
3762 device->rescan = true;
3763 }
3764
3765 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
3766}
3767
3768static void pqi_disable_raid_bypass(struct pqi_ctrl_info *ctrl_info)
3769{
3770 unsigned long flags;
3771 struct pqi_scsi_dev *device;
3772
3773 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
3774
3775 list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry)
3776 if (device->raid_bypass_enabled)
3777 device->raid_bypass_enabled = false;
3778
3779 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
3780}
3781
3782static void pqi_event_worker(struct work_struct *work)
3783{
3784 unsigned int i;
3785 bool rescan_needed;
3786 struct pqi_ctrl_info *ctrl_info;
3787 struct pqi_event *event;
3788 bool ack_event;
3789
3790 ctrl_info = container_of(work, struct pqi_ctrl_info, event_work);
3791
3792 pqi_ctrl_busy(ctrl_info);
3793 pqi_wait_if_ctrl_blocked(ctrl_info);
3794 if (pqi_ctrl_offline(ctrl_info))
3795 goto out;
3796
3797 rescan_needed = false;
3798 event = ctrl_info->events;
3799 for (i = 0; i < PQI_NUM_SUPPORTED_EVENTS; i++) {
3800 if (event->pending) {
3801 event->pending = false;
3802 if (event->event_type == PQI_EVENT_TYPE_OFA) {
3803 ack_event = pqi_ofa_process_event(ctrl_info, event);
3804 } else {
3805 ack_event = true;
3806 rescan_needed = true;
3807 if (event->event_type == PQI_EVENT_TYPE_LOGICAL_DEVICE)
3808 pqi_mark_volumes_for_rescan(ctrl_info);
3809 else if (event->event_type == PQI_EVENT_TYPE_AIO_STATE_CHANGE)
3810 pqi_disable_raid_bypass(ctrl_info);
3811 }
3812 if (ack_event)
3813 pqi_acknowledge_event(ctrl_info, event);
3814 }
3815 event++;
3816 }
3817
3818#define PQI_RESCAN_WORK_FOR_EVENT_DELAY (5 * HZ)
3819
3820 if (rescan_needed)
3821 pqi_schedule_rescan_worker_with_delay(ctrl_info,
3822 PQI_RESCAN_WORK_FOR_EVENT_DELAY);
3823
3824out:
3825 pqi_ctrl_unbusy(ctrl_info);
3826}
3827
3828#define PQI_HEARTBEAT_TIMER_INTERVAL (10 * HZ)
3829
3830static void pqi_heartbeat_timer_handler(struct timer_list *t)
3831{
3832 int num_interrupts;
3833 u32 heartbeat_count;
3834 struct pqi_ctrl_info *ctrl_info = timer_container_of(ctrl_info, t,
3835 heartbeat_timer);
3836
3837 pqi_check_ctrl_health(ctrl_info);
3838 if (pqi_ctrl_offline(ctrl_info))
3839 return;
3840
3841 num_interrupts = atomic_read(&ctrl_info->num_interrupts);
3842 heartbeat_count = pqi_read_heartbeat_counter(ctrl_info);
3843
3844 if (num_interrupts == ctrl_info->previous_num_interrupts) {
3845 if (heartbeat_count == ctrl_info->previous_heartbeat_count) {
3846 dev_err(&ctrl_info->pci_dev->dev,
3847 "no heartbeat detected - last heartbeat count: %u\n",
3848 heartbeat_count);
3849 pqi_take_ctrl_offline(ctrl_info, PQI_NO_HEARTBEAT);
3850 return;
3851 }
3852 } else {
3853 ctrl_info->previous_num_interrupts = num_interrupts;
3854 }
3855
3856 ctrl_info->previous_heartbeat_count = heartbeat_count;
3857 mod_timer(&ctrl_info->heartbeat_timer,
3858 jiffies + PQI_HEARTBEAT_TIMER_INTERVAL);
3859}
3860
3861static void pqi_start_heartbeat_timer(struct pqi_ctrl_info *ctrl_info)
3862{
3863 if (!ctrl_info->heartbeat_counter)
3864 return;
3865
3866 ctrl_info->previous_num_interrupts =
3867 atomic_read(&ctrl_info->num_interrupts);
3868 ctrl_info->previous_heartbeat_count =
3869 pqi_read_heartbeat_counter(ctrl_info);
3870
3871 ctrl_info->heartbeat_timer.expires =
3872 jiffies + PQI_HEARTBEAT_TIMER_INTERVAL;
3873 add_timer(&ctrl_info->heartbeat_timer);
3874}
3875
3876static inline void pqi_stop_heartbeat_timer(struct pqi_ctrl_info *ctrl_info)
3877{
3878 timer_delete_sync(&ctrl_info->heartbeat_timer);
3879}
3880
3881static void pqi_ofa_capture_event_payload(struct pqi_ctrl_info *ctrl_info,
3882 struct pqi_event *event, struct pqi_event_response *response)
3883{
3884 switch (event->event_id) {
3885 case PQI_EVENT_OFA_MEMORY_ALLOCATION:
3886 ctrl_info->ofa_bytes_requested =
3887 get_unaligned_le32(&response->data.ofa_memory_allocation.bytes_requested);
3888 break;
3889 case PQI_EVENT_OFA_CANCELED:
3890 ctrl_info->ofa_cancel_reason =
3891 get_unaligned_le16(&response->data.ofa_cancelled.reason);
3892 break;
3893 }
3894}
3895
3896static int pqi_process_event_intr(struct pqi_ctrl_info *ctrl_info)
3897{
3898 int num_events;
3899 pqi_index_t oq_pi;
3900 pqi_index_t oq_ci;
3901 struct pqi_event_queue *event_queue;
3902 struct pqi_event_response *response;
3903 struct pqi_event *event;
3904 int event_index;
3905
3906 event_queue = &ctrl_info->event_queue;
3907 num_events = 0;
3908 oq_ci = event_queue->oq_ci_copy;
3909
3910 while (1) {
3911 oq_pi = readl(event_queue->oq_pi);
3912 if (oq_pi >= PQI_NUM_EVENT_QUEUE_ELEMENTS) {
3913 pqi_invalid_response(ctrl_info, PQI_EVENT_PI_OUT_OF_RANGE);
3914 dev_err(&ctrl_info->pci_dev->dev,
3915 "event interrupt: producer index (%u) out of range (0-%u): consumer index: %u\n",
3916 oq_pi, PQI_NUM_EVENT_QUEUE_ELEMENTS - 1, oq_ci);
3917 return -1;
3918 }
3919
3920 if (oq_pi == oq_ci)
3921 break;
3922
3923 num_events++;
3924 response = event_queue->oq_element_array + (oq_ci * PQI_EVENT_OQ_ELEMENT_LENGTH);
3925
3926 event_index = pqi_event_type_to_event_index(response->event_type);
3927
3928 if (event_index >= 0 && response->request_acknowledge) {
3929 event = &ctrl_info->events[event_index];
3930 event->pending = true;
3931 event->event_type = response->event_type;
3932 event->event_id = get_unaligned_le16(&response->event_id);
3933 event->additional_event_id =
3934 get_unaligned_le32(&response->additional_event_id);
3935 if (event->event_type == PQI_EVENT_TYPE_OFA)
3936 pqi_ofa_capture_event_payload(ctrl_info, event, response);
3937 }
3938
3939 oq_ci = (oq_ci + 1) % PQI_NUM_EVENT_QUEUE_ELEMENTS;
3940 }
3941
3942 if (num_events) {
3943 event_queue->oq_ci_copy = oq_ci;
3944 writel(oq_ci, event_queue->oq_ci);
3945 schedule_work(&ctrl_info->event_work);
3946 }
3947
3948 return num_events;
3949}
3950
3951#define PQI_LEGACY_INTX_MASK 0x1
3952
3953static inline void pqi_configure_legacy_intx(struct pqi_ctrl_info *ctrl_info, bool enable_intx)
3954{
3955 u32 intx_mask;
3956 struct pqi_device_registers __iomem *pqi_registers;
3957 volatile void __iomem *register_addr;
3958
3959 pqi_registers = ctrl_info->pqi_registers;
3960
3961 if (enable_intx)
3962 register_addr = &pqi_registers->legacy_intx_mask_clear;
3963 else
3964 register_addr = &pqi_registers->legacy_intx_mask_set;
3965
3966 intx_mask = readl(register_addr);
3967 intx_mask |= PQI_LEGACY_INTX_MASK;
3968 writel(intx_mask, register_addr);
3969}
3970
3971static void pqi_change_irq_mode(struct pqi_ctrl_info *ctrl_info,
3972 enum pqi_irq_mode new_mode)
3973{
3974 switch (ctrl_info->irq_mode) {
3975 case IRQ_MODE_MSIX:
3976 switch (new_mode) {
3977 case IRQ_MODE_MSIX:
3978 break;
3979 case IRQ_MODE_INTX:
3980 pqi_configure_legacy_intx(ctrl_info, true);
3981 sis_enable_intx(ctrl_info);
3982 break;
3983 case IRQ_MODE_NONE:
3984 break;
3985 }
3986 break;
3987 case IRQ_MODE_INTX:
3988 switch (new_mode) {
3989 case IRQ_MODE_MSIX:
3990 pqi_configure_legacy_intx(ctrl_info, false);
3991 sis_enable_msix(ctrl_info);
3992 break;
3993 case IRQ_MODE_INTX:
3994 break;
3995 case IRQ_MODE_NONE:
3996 pqi_configure_legacy_intx(ctrl_info, false);
3997 break;
3998 }
3999 break;
4000 case IRQ_MODE_NONE:
4001 switch (new_mode) {
4002 case IRQ_MODE_MSIX:
4003 sis_enable_msix(ctrl_info);
4004 break;
4005 case IRQ_MODE_INTX:
4006 pqi_configure_legacy_intx(ctrl_info, true);
4007 sis_enable_intx(ctrl_info);
4008 break;
4009 case IRQ_MODE_NONE:
4010 break;
4011 }
4012 break;
4013 }
4014
4015 ctrl_info->irq_mode = new_mode;
4016}
4017
4018#define PQI_LEGACY_INTX_PENDING 0x1
4019
4020static inline bool pqi_is_valid_irq(struct pqi_ctrl_info *ctrl_info)
4021{
4022 bool valid_irq;
4023 u32 intx_status;
4024
4025 switch (ctrl_info->irq_mode) {
4026 case IRQ_MODE_MSIX:
4027 valid_irq = true;
4028 break;
4029 case IRQ_MODE_INTX:
4030 intx_status = readl(&ctrl_info->pqi_registers->legacy_intx_status);
4031 if (intx_status & PQI_LEGACY_INTX_PENDING)
4032 valid_irq = true;
4033 else
4034 valid_irq = false;
4035 break;
4036 case IRQ_MODE_NONE:
4037 default:
4038 valid_irq = false;
4039 break;
4040 }
4041
4042 return valid_irq;
4043}
4044
4045static irqreturn_t pqi_irq_handler(int irq, void *data)
4046{
4047 struct pqi_ctrl_info *ctrl_info;
4048 struct pqi_queue_group *queue_group;
4049 int num_io_responses_handled;
4050 int num_events_handled;
4051
4052 queue_group = data;
4053 ctrl_info = queue_group->ctrl_info;
4054
4055 if (!pqi_is_valid_irq(ctrl_info))
4056 return IRQ_NONE;
4057
4058 num_io_responses_handled = pqi_process_io_intr(ctrl_info, queue_group);
4059 if (num_io_responses_handled < 0)
4060 goto out;
4061
4062 if (irq == ctrl_info->event_irq) {
4063 num_events_handled = pqi_process_event_intr(ctrl_info);
4064 if (num_events_handled < 0)
4065 goto out;
4066 } else {
4067 num_events_handled = 0;
4068 }
4069
4070 if (num_io_responses_handled + num_events_handled > 0)
4071 atomic_inc(&ctrl_info->num_interrupts);
4072
4073 pqi_start_io(ctrl_info, queue_group, RAID_PATH, NULL);
4074 pqi_start_io(ctrl_info, queue_group, AIO_PATH, NULL);
4075
4076out:
4077 return IRQ_HANDLED;
4078}
4079
4080static int pqi_request_irqs(struct pqi_ctrl_info *ctrl_info)
4081{
4082 struct pci_dev *pci_dev = ctrl_info->pci_dev;
4083 int i;
4084 int rc;
4085
4086 ctrl_info->event_irq = pci_irq_vector(pci_dev, 0);
4087
4088 for (i = 0; i < ctrl_info->num_msix_vectors_enabled; i++) {
4089 rc = request_irq(pci_irq_vector(pci_dev, i), pqi_irq_handler, 0,
4090 DRIVER_NAME_SHORT, &ctrl_info->queue_groups[i]);
4091 if (rc) {
4092 dev_err(&pci_dev->dev,
4093 "irq %u init failed with error %d\n",
4094 pci_irq_vector(pci_dev, i), rc);
4095 return rc;
4096 }
4097 ctrl_info->num_msix_vectors_initialized++;
4098 }
4099
4100 return 0;
4101}
4102
4103static void pqi_free_irqs(struct pqi_ctrl_info *ctrl_info)
4104{
4105 int i;
4106
4107 for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++)
4108 free_irq(pci_irq_vector(ctrl_info->pci_dev, i),
4109 &ctrl_info->queue_groups[i]);
4110
4111 ctrl_info->num_msix_vectors_initialized = 0;
4112}
4113
4114static int pqi_enable_msix_interrupts(struct pqi_ctrl_info *ctrl_info)
4115{
4116 int num_vectors_enabled;
4117 unsigned int flags = PCI_IRQ_MSIX;
4118
4119 if (!pqi_disable_managed_interrupts)
4120 flags |= PCI_IRQ_AFFINITY;
4121
4122 num_vectors_enabled = pci_alloc_irq_vectors(ctrl_info->pci_dev,
4123 PQI_MIN_MSIX_VECTORS, ctrl_info->num_queue_groups,
4124 flags);
4125 if (num_vectors_enabled < 0) {
4126 dev_err(&ctrl_info->pci_dev->dev,
4127 "MSI-X init failed with error %d\n",
4128 num_vectors_enabled);
4129 return num_vectors_enabled;
4130 }
4131
4132 ctrl_info->num_msix_vectors_enabled = num_vectors_enabled;
4133 ctrl_info->irq_mode = IRQ_MODE_MSIX;
4134 return 0;
4135}
4136
4137static void pqi_disable_msix_interrupts(struct pqi_ctrl_info *ctrl_info)
4138{
4139 if (ctrl_info->num_msix_vectors_enabled) {
4140 pci_free_irq_vectors(ctrl_info->pci_dev);
4141 ctrl_info->num_msix_vectors_enabled = 0;
4142 }
4143}
4144
4145static int pqi_alloc_operational_queues(struct pqi_ctrl_info *ctrl_info)
4146{
4147 unsigned int i;
4148 size_t alloc_length;
4149 size_t element_array_length_per_iq;
4150 size_t element_array_length_per_oq;
4151 void *element_array;
4152 void __iomem *next_queue_index;
4153 void *aligned_pointer;
4154 unsigned int num_inbound_queues;
4155 unsigned int num_outbound_queues;
4156 unsigned int num_queue_indexes;
4157 struct pqi_queue_group *queue_group;
4158
4159 element_array_length_per_iq =
4160 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH *
4161 ctrl_info->num_elements_per_iq;
4162 element_array_length_per_oq =
4163 PQI_OPERATIONAL_OQ_ELEMENT_LENGTH *
4164 ctrl_info->num_elements_per_oq;
4165 num_inbound_queues = ctrl_info->num_queue_groups * 2;
4166 num_outbound_queues = ctrl_info->num_queue_groups;
4167 num_queue_indexes = (ctrl_info->num_queue_groups * 3) + 1;
4168
4169 aligned_pointer = NULL;
4170
4171 for (i = 0; i < num_inbound_queues; i++) {
4172 aligned_pointer = PTR_ALIGN(aligned_pointer,
4173 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
4174 aligned_pointer += element_array_length_per_iq;
4175 }
4176
4177 for (i = 0; i < num_outbound_queues; i++) {
4178 aligned_pointer = PTR_ALIGN(aligned_pointer,
4179 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
4180 aligned_pointer += element_array_length_per_oq;
4181 }
4182
4183 aligned_pointer = PTR_ALIGN(aligned_pointer,
4184 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
4185 aligned_pointer += PQI_NUM_EVENT_QUEUE_ELEMENTS *
4186 PQI_EVENT_OQ_ELEMENT_LENGTH;
4187
4188 for (i = 0; i < num_queue_indexes; i++) {
4189 aligned_pointer = PTR_ALIGN(aligned_pointer,
4190 PQI_OPERATIONAL_INDEX_ALIGNMENT);
4191 aligned_pointer += sizeof(pqi_index_t);
4192 }
4193
4194 alloc_length = (size_t)aligned_pointer +
4195 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT;
4196
4197 alloc_length += PQI_EXTRA_SGL_MEMORY;
4198
4199 ctrl_info->queue_memory_base =
4200 dma_alloc_coherent(&ctrl_info->pci_dev->dev, alloc_length,
4201 &ctrl_info->queue_memory_base_dma_handle,
4202 GFP_KERNEL);
4203
4204 if (!ctrl_info->queue_memory_base)
4205 return -ENOMEM;
4206
4207 ctrl_info->queue_memory_length = alloc_length;
4208
4209 element_array = PTR_ALIGN(ctrl_info->queue_memory_base,
4210 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
4211
4212 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
4213 queue_group = &ctrl_info->queue_groups[i];
4214 queue_group->iq_element_array[RAID_PATH] = element_array;
4215 queue_group->iq_element_array_bus_addr[RAID_PATH] =
4216 ctrl_info->queue_memory_base_dma_handle +
4217 (element_array - ctrl_info->queue_memory_base);
4218 element_array += element_array_length_per_iq;
4219 element_array = PTR_ALIGN(element_array,
4220 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
4221 queue_group->iq_element_array[AIO_PATH] = element_array;
4222 queue_group->iq_element_array_bus_addr[AIO_PATH] =
4223 ctrl_info->queue_memory_base_dma_handle +
4224 (element_array - ctrl_info->queue_memory_base);
4225 element_array += element_array_length_per_iq;
4226 element_array = PTR_ALIGN(element_array,
4227 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
4228 }
4229
4230 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
4231 queue_group = &ctrl_info->queue_groups[i];
4232 queue_group->oq_element_array = element_array;
4233 queue_group->oq_element_array_bus_addr =
4234 ctrl_info->queue_memory_base_dma_handle +
4235 (element_array - ctrl_info->queue_memory_base);
4236 element_array += element_array_length_per_oq;
4237 element_array = PTR_ALIGN(element_array,
4238 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
4239 }
4240
4241 ctrl_info->event_queue.oq_element_array = element_array;
4242 ctrl_info->event_queue.oq_element_array_bus_addr =
4243 ctrl_info->queue_memory_base_dma_handle +
4244 (element_array - ctrl_info->queue_memory_base);
4245 element_array += PQI_NUM_EVENT_QUEUE_ELEMENTS *
4246 PQI_EVENT_OQ_ELEMENT_LENGTH;
4247
4248 next_queue_index = (void __iomem *)PTR_ALIGN(element_array,
4249 PQI_OPERATIONAL_INDEX_ALIGNMENT);
4250
4251 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
4252 queue_group = &ctrl_info->queue_groups[i];
4253 queue_group->iq_ci[RAID_PATH] = next_queue_index;
4254 queue_group->iq_ci_bus_addr[RAID_PATH] =
4255 ctrl_info->queue_memory_base_dma_handle +
4256 (next_queue_index -
4257 (void __iomem *)ctrl_info->queue_memory_base);
4258 next_queue_index += sizeof(pqi_index_t);
4259 next_queue_index = PTR_ALIGN(next_queue_index,
4260 PQI_OPERATIONAL_INDEX_ALIGNMENT);
4261 queue_group->iq_ci[AIO_PATH] = next_queue_index;
4262 queue_group->iq_ci_bus_addr[AIO_PATH] =
4263 ctrl_info->queue_memory_base_dma_handle +
4264 (next_queue_index -
4265 (void __iomem *)ctrl_info->queue_memory_base);
4266 next_queue_index += sizeof(pqi_index_t);
4267 next_queue_index = PTR_ALIGN(next_queue_index,
4268 PQI_OPERATIONAL_INDEX_ALIGNMENT);
4269 queue_group->oq_pi = next_queue_index;
4270 queue_group->oq_pi_bus_addr =
4271 ctrl_info->queue_memory_base_dma_handle +
4272 (next_queue_index -
4273 (void __iomem *)ctrl_info->queue_memory_base);
4274 next_queue_index += sizeof(pqi_index_t);
4275 next_queue_index = PTR_ALIGN(next_queue_index,
4276 PQI_OPERATIONAL_INDEX_ALIGNMENT);
4277 }
4278
4279 ctrl_info->event_queue.oq_pi = next_queue_index;
4280 ctrl_info->event_queue.oq_pi_bus_addr =
4281 ctrl_info->queue_memory_base_dma_handle +
4282 (next_queue_index -
4283 (void __iomem *)ctrl_info->queue_memory_base);
4284
4285 return 0;
4286}
4287
4288static void pqi_init_operational_queues(struct pqi_ctrl_info *ctrl_info)
4289{
4290 unsigned int i;
4291 u16 next_iq_id = PQI_MIN_OPERATIONAL_QUEUE_ID;
4292 u16 next_oq_id = PQI_MIN_OPERATIONAL_QUEUE_ID;
4293
4294 /*
4295 * Initialize the backpointers to the controller structure in
4296 * each operational queue group structure.
4297 */
4298 for (i = 0; i < ctrl_info->num_queue_groups; i++)
4299 ctrl_info->queue_groups[i].ctrl_info = ctrl_info;
4300
4301 /*
4302 * Assign IDs to all operational queues. Note that the IDs
4303 * assigned to operational IQs are independent of the IDs
4304 * assigned to operational OQs.
4305 */
4306 ctrl_info->event_queue.oq_id = next_oq_id++;
4307 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
4308 ctrl_info->queue_groups[i].iq_id[RAID_PATH] = next_iq_id++;
4309 ctrl_info->queue_groups[i].iq_id[AIO_PATH] = next_iq_id++;
4310 ctrl_info->queue_groups[i].oq_id = next_oq_id++;
4311 }
4312
4313 /*
4314 * Assign MSI-X table entry indexes to all queues. Note that the
4315 * interrupt for the event queue is shared with the first queue group.
4316 */
4317 ctrl_info->event_queue.int_msg_num = 0;
4318 for (i = 0; i < ctrl_info->num_queue_groups; i++)
4319 ctrl_info->queue_groups[i].int_msg_num = i;
4320
4321 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
4322 spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[0]);
4323 spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[1]);
4324 INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[0]);
4325 INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[1]);
4326 }
4327}
4328
4329static int pqi_alloc_admin_queues(struct pqi_ctrl_info *ctrl_info)
4330{
4331 size_t alloc_length;
4332 struct pqi_admin_queues_aligned *admin_queues_aligned;
4333 struct pqi_admin_queues *admin_queues;
4334
4335 alloc_length = sizeof(struct pqi_admin_queues_aligned) +
4336 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT;
4337
4338 ctrl_info->admin_queue_memory_base =
4339 dma_alloc_coherent(&ctrl_info->pci_dev->dev, alloc_length,
4340 &ctrl_info->admin_queue_memory_base_dma_handle,
4341 GFP_KERNEL);
4342
4343 if (!ctrl_info->admin_queue_memory_base)
4344 return -ENOMEM;
4345
4346 ctrl_info->admin_queue_memory_length = alloc_length;
4347
4348 admin_queues = &ctrl_info->admin_queues;
4349 admin_queues_aligned = PTR_ALIGN(ctrl_info->admin_queue_memory_base,
4350 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
4351 admin_queues->iq_element_array =
4352 &admin_queues_aligned->iq_element_array;
4353 admin_queues->oq_element_array =
4354 &admin_queues_aligned->oq_element_array;
4355 admin_queues->iq_ci =
4356 (pqi_index_t __iomem *)&admin_queues_aligned->iq_ci;
4357 admin_queues->oq_pi =
4358 (pqi_index_t __iomem *)&admin_queues_aligned->oq_pi;
4359
4360 admin_queues->iq_element_array_bus_addr =
4361 ctrl_info->admin_queue_memory_base_dma_handle +
4362 (admin_queues->iq_element_array -
4363 ctrl_info->admin_queue_memory_base);
4364 admin_queues->oq_element_array_bus_addr =
4365 ctrl_info->admin_queue_memory_base_dma_handle +
4366 (admin_queues->oq_element_array -
4367 ctrl_info->admin_queue_memory_base);
4368 admin_queues->iq_ci_bus_addr =
4369 ctrl_info->admin_queue_memory_base_dma_handle +
4370 ((void __iomem *)admin_queues->iq_ci -
4371 (void __iomem *)ctrl_info->admin_queue_memory_base);
4372 admin_queues->oq_pi_bus_addr =
4373 ctrl_info->admin_queue_memory_base_dma_handle +
4374 ((void __iomem *)admin_queues->oq_pi -
4375 (void __iomem *)ctrl_info->admin_queue_memory_base);
4376
4377 return 0;
4378}
4379
4380#define PQI_ADMIN_QUEUE_CREATE_TIMEOUT_JIFFIES HZ
4381#define PQI_ADMIN_QUEUE_CREATE_POLL_INTERVAL_MSECS 1
4382
4383static int pqi_create_admin_queues(struct pqi_ctrl_info *ctrl_info)
4384{
4385 struct pqi_device_registers __iomem *pqi_registers;
4386 struct pqi_admin_queues *admin_queues;
4387 unsigned long timeout;
4388 u8 status;
4389 u32 reg;
4390
4391 pqi_registers = ctrl_info->pqi_registers;
4392 admin_queues = &ctrl_info->admin_queues;
4393
4394 writeq((u64)admin_queues->iq_element_array_bus_addr,
4395 &pqi_registers->admin_iq_element_array_addr);
4396 writeq((u64)admin_queues->oq_element_array_bus_addr,
4397 &pqi_registers->admin_oq_element_array_addr);
4398 writeq((u64)admin_queues->iq_ci_bus_addr,
4399 &pqi_registers->admin_iq_ci_addr);
4400 writeq((u64)admin_queues->oq_pi_bus_addr,
4401 &pqi_registers->admin_oq_pi_addr);
4402
4403 reg = PQI_ADMIN_IQ_NUM_ELEMENTS |
4404 (PQI_ADMIN_OQ_NUM_ELEMENTS << 8) |
4405 (admin_queues->int_msg_num << 16);
4406 writel(reg, &pqi_registers->admin_iq_num_elements);
4407
4408 writel(PQI_CREATE_ADMIN_QUEUE_PAIR,
4409 &pqi_registers->function_and_status_code);
4410
4411 timeout = PQI_ADMIN_QUEUE_CREATE_TIMEOUT_JIFFIES + jiffies;
4412 while (1) {
4413 msleep(PQI_ADMIN_QUEUE_CREATE_POLL_INTERVAL_MSECS);
4414 status = readb(&pqi_registers->function_and_status_code);
4415 if (status == PQI_STATUS_IDLE)
4416 break;
4417 if (time_after(jiffies, timeout))
4418 return -ETIMEDOUT;
4419 }
4420
4421 /*
4422 * The offset registers are not initialized to the correct
4423 * offsets until *after* the create admin queue pair command
4424 * completes successfully.
4425 */
4426 admin_queues->iq_pi = ctrl_info->iomem_base +
4427 PQI_DEVICE_REGISTERS_OFFSET +
4428 readq(&pqi_registers->admin_iq_pi_offset);
4429 admin_queues->oq_ci = ctrl_info->iomem_base +
4430 PQI_DEVICE_REGISTERS_OFFSET +
4431 readq(&pqi_registers->admin_oq_ci_offset);
4432
4433 return 0;
4434}
4435
4436static void pqi_submit_admin_request(struct pqi_ctrl_info *ctrl_info,
4437 struct pqi_general_admin_request *request)
4438{
4439 struct pqi_admin_queues *admin_queues;
4440 void *next_element;
4441 pqi_index_t iq_pi;
4442
4443 admin_queues = &ctrl_info->admin_queues;
4444 iq_pi = admin_queues->iq_pi_copy;
4445
4446 next_element = admin_queues->iq_element_array +
4447 (iq_pi * PQI_ADMIN_IQ_ELEMENT_LENGTH);
4448
4449 memcpy(next_element, request, sizeof(*request));
4450
4451 iq_pi = (iq_pi + 1) % PQI_ADMIN_IQ_NUM_ELEMENTS;
4452 admin_queues->iq_pi_copy = iq_pi;
4453
4454 /*
4455 * This write notifies the controller that an IU is available to be
4456 * processed.
4457 */
4458 writel(iq_pi, admin_queues->iq_pi);
4459}
4460
4461#define PQI_ADMIN_REQUEST_TIMEOUT_SECS 60
4462
4463static int pqi_poll_for_admin_response(struct pqi_ctrl_info *ctrl_info,
4464 struct pqi_general_admin_response *response)
4465{
4466 struct pqi_admin_queues *admin_queues;
4467 pqi_index_t oq_pi;
4468 pqi_index_t oq_ci;
4469 unsigned long timeout;
4470
4471 admin_queues = &ctrl_info->admin_queues;
4472 oq_ci = admin_queues->oq_ci_copy;
4473
4474 timeout = (PQI_ADMIN_REQUEST_TIMEOUT_SECS * HZ) + jiffies;
4475
4476 while (1) {
4477 oq_pi = readl(admin_queues->oq_pi);
4478 if (oq_pi != oq_ci)
4479 break;
4480 if (time_after(jiffies, timeout)) {
4481 dev_err(&ctrl_info->pci_dev->dev,
4482 "timed out waiting for admin response\n");
4483 return -ETIMEDOUT;
4484 }
4485 if (!sis_is_firmware_running(ctrl_info))
4486 return -ENXIO;
4487 usleep_range(1000, 2000);
4488 }
4489
4490 memcpy(response, admin_queues->oq_element_array +
4491 (oq_ci * PQI_ADMIN_OQ_ELEMENT_LENGTH), sizeof(*response));
4492
4493 oq_ci = (oq_ci + 1) % PQI_ADMIN_OQ_NUM_ELEMENTS;
4494 admin_queues->oq_ci_copy = oq_ci;
4495 writel(oq_ci, admin_queues->oq_ci);
4496
4497 return 0;
4498}
4499
4500static void pqi_start_io(struct pqi_ctrl_info *ctrl_info,
4501 struct pqi_queue_group *queue_group, enum pqi_io_path path,
4502 struct pqi_io_request *io_request)
4503{
4504 struct pqi_io_request *next;
4505 void *next_element;
4506 pqi_index_t iq_pi;
4507 pqi_index_t iq_ci;
4508 size_t iu_length;
4509 unsigned long flags;
4510 unsigned int num_elements_needed;
4511 unsigned int num_elements_to_end_of_queue;
4512 size_t copy_count;
4513 struct pqi_iu_header *request;
4514
4515 spin_lock_irqsave(&queue_group->submit_lock[path], flags);
4516
4517 if (io_request) {
4518 io_request->queue_group = queue_group;
4519 list_add_tail(&io_request->request_list_entry,
4520 &queue_group->request_list[path]);
4521 }
4522
4523 iq_pi = queue_group->iq_pi_copy[path];
4524
4525 list_for_each_entry_safe(io_request, next,
4526 &queue_group->request_list[path], request_list_entry) {
4527
4528 request = io_request->iu;
4529
4530 iu_length = get_unaligned_le16(&request->iu_length) +
4531 PQI_REQUEST_HEADER_LENGTH;
4532 num_elements_needed =
4533 DIV_ROUND_UP(iu_length,
4534 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
4535
4536 iq_ci = readl(queue_group->iq_ci[path]);
4537
4538 if (num_elements_needed > pqi_num_elements_free(iq_pi, iq_ci,
4539 ctrl_info->num_elements_per_iq))
4540 break;
4541
4542 put_unaligned_le16(queue_group->oq_id,
4543 &request->response_queue_id);
4544
4545 next_element = queue_group->iq_element_array[path] +
4546 (iq_pi * PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
4547
4548 num_elements_to_end_of_queue =
4549 ctrl_info->num_elements_per_iq - iq_pi;
4550
4551 if (num_elements_needed <= num_elements_to_end_of_queue) {
4552 memcpy(next_element, request, iu_length);
4553 } else {
4554 copy_count = num_elements_to_end_of_queue *
4555 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH;
4556 memcpy(next_element, request, copy_count);
4557 memcpy(queue_group->iq_element_array[path],
4558 (u8 *)request + copy_count,
4559 iu_length - copy_count);
4560 }
4561
4562 iq_pi = (iq_pi + num_elements_needed) %
4563 ctrl_info->num_elements_per_iq;
4564
4565 list_del(&io_request->request_list_entry);
4566 }
4567
4568 if (iq_pi != queue_group->iq_pi_copy[path]) {
4569 queue_group->iq_pi_copy[path] = iq_pi;
4570 /*
4571 * This write notifies the controller that one or more IUs are
4572 * available to be processed.
4573 */
4574 writel(iq_pi, queue_group->iq_pi[path]);
4575 }
4576
4577 spin_unlock_irqrestore(&queue_group->submit_lock[path], flags);
4578}
4579
4580#define PQI_WAIT_FOR_COMPLETION_IO_TIMEOUT_SECS 10
4581
4582static int pqi_wait_for_completion_io(struct pqi_ctrl_info *ctrl_info,
4583 struct completion *wait)
4584{
4585 int rc;
4586
4587 while (1) {
4588 if (wait_for_completion_io_timeout(wait,
4589 PQI_WAIT_FOR_COMPLETION_IO_TIMEOUT_SECS * HZ)) {
4590 rc = 0;
4591 break;
4592 }
4593
4594 pqi_check_ctrl_health(ctrl_info);
4595 if (pqi_ctrl_offline(ctrl_info)) {
4596 rc = -ENXIO;
4597 break;
4598 }
4599 }
4600
4601 return rc;
4602}
4603
4604static void pqi_raid_synchronous_complete(struct pqi_io_request *io_request,
4605 void *context)
4606{
4607 struct completion *waiting = context;
4608
4609 complete(waiting);
4610}
4611
4612static int pqi_process_raid_io_error_synchronous(
4613 struct pqi_raid_error_info *error_info)
4614{
4615 int rc = -EIO;
4616
4617 switch (error_info->data_out_result) {
4618 case PQI_DATA_IN_OUT_GOOD:
4619 if (error_info->status == SAM_STAT_GOOD)
4620 rc = 0;
4621 break;
4622 case PQI_DATA_IN_OUT_UNDERFLOW:
4623 if (error_info->status == SAM_STAT_GOOD ||
4624 error_info->status == SAM_STAT_CHECK_CONDITION)
4625 rc = 0;
4626 break;
4627 case PQI_DATA_IN_OUT_ABORTED:
4628 rc = PQI_CMD_STATUS_ABORTED;
4629 break;
4630 }
4631
4632 return rc;
4633}
4634
4635static inline bool pqi_is_blockable_request(struct pqi_iu_header *request)
4636{
4637 return (request->driver_flags & PQI_DRIVER_NONBLOCKABLE_REQUEST) == 0;
4638}
4639
4640static int pqi_submit_raid_request_synchronous(struct pqi_ctrl_info *ctrl_info,
4641 struct pqi_iu_header *request, unsigned int flags,
4642 struct pqi_raid_error_info *error_info)
4643{
4644 int rc = 0;
4645 struct pqi_io_request *io_request;
4646 size_t iu_length;
4647 DECLARE_COMPLETION_ONSTACK(wait);
4648
4649 if (flags & PQI_SYNC_FLAGS_INTERRUPTABLE) {
4650 if (down_interruptible(&ctrl_info->sync_request_sem))
4651 return -ERESTARTSYS;
4652 } else {
4653 down(&ctrl_info->sync_request_sem);
4654 }
4655
4656 pqi_ctrl_busy(ctrl_info);
4657 /*
4658 * Wait for other admin queue updates such as;
4659 * config table changes, OFA memory updates, ...
4660 */
4661 if (pqi_is_blockable_request(request))
4662 pqi_wait_if_ctrl_blocked(ctrl_info);
4663
4664 if (pqi_ctrl_offline(ctrl_info)) {
4665 rc = -ENXIO;
4666 goto out;
4667 }
4668
4669 io_request = pqi_alloc_io_request(ctrl_info, NULL);
4670
4671 put_unaligned_le16(io_request->index,
4672 &(((struct pqi_raid_path_request *)request)->request_id));
4673
4674 if (request->iu_type == PQI_REQUEST_IU_RAID_PATH_IO)
4675 ((struct pqi_raid_path_request *)request)->error_index =
4676 ((struct pqi_raid_path_request *)request)->request_id;
4677
4678 iu_length = get_unaligned_le16(&request->iu_length) +
4679 PQI_REQUEST_HEADER_LENGTH;
4680 memcpy(io_request->iu, request, iu_length);
4681
4682 io_request->io_complete_callback = pqi_raid_synchronous_complete;
4683 io_request->context = &wait;
4684
4685 pqi_start_io(ctrl_info, &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,
4686 io_request);
4687
4688 pqi_wait_for_completion_io(ctrl_info, &wait);
4689
4690 if (error_info) {
4691 if (io_request->error_info)
4692 memcpy(error_info, io_request->error_info, sizeof(*error_info));
4693 else
4694 memset(error_info, 0, sizeof(*error_info));
4695 } else if (rc == 0 && io_request->error_info) {
4696 rc = pqi_process_raid_io_error_synchronous(io_request->error_info);
4697 }
4698
4699 pqi_free_io_request(io_request);
4700
4701out:
4702 pqi_ctrl_unbusy(ctrl_info);
4703 up(&ctrl_info->sync_request_sem);
4704
4705 return rc;
4706}
4707
4708static int pqi_validate_admin_response(
4709 struct pqi_general_admin_response *response, u8 expected_function_code)
4710{
4711 if (response->header.iu_type != PQI_RESPONSE_IU_GENERAL_ADMIN)
4712 return -EINVAL;
4713
4714 if (get_unaligned_le16(&response->header.iu_length) !=
4715 PQI_GENERAL_ADMIN_IU_LENGTH)
4716 return -EINVAL;
4717
4718 if (response->function_code != expected_function_code)
4719 return -EINVAL;
4720
4721 if (response->status != PQI_GENERAL_ADMIN_STATUS_SUCCESS)
4722 return -EINVAL;
4723
4724 return 0;
4725}
4726
4727static int pqi_submit_admin_request_synchronous(
4728 struct pqi_ctrl_info *ctrl_info,
4729 struct pqi_general_admin_request *request,
4730 struct pqi_general_admin_response *response)
4731{
4732 int rc;
4733
4734 pqi_submit_admin_request(ctrl_info, request);
4735
4736 rc = pqi_poll_for_admin_response(ctrl_info, response);
4737
4738 if (rc == 0)
4739 rc = pqi_validate_admin_response(response, request->function_code);
4740
4741 return rc;
4742}
4743
4744static int pqi_report_device_capability(struct pqi_ctrl_info *ctrl_info)
4745{
4746 int rc;
4747 struct pqi_general_admin_request request;
4748 struct pqi_general_admin_response response;
4749 struct pqi_device_capability *capability;
4750 struct pqi_iu_layer_descriptor *sop_iu_layer_descriptor;
4751
4752 capability = kmalloc_obj(*capability);
4753 if (!capability)
4754 return -ENOMEM;
4755
4756 memset(&request, 0, sizeof(request));
4757
4758 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4759 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4760 &request.header.iu_length);
4761 request.function_code =
4762 PQI_GENERAL_ADMIN_FUNCTION_REPORT_DEVICE_CAPABILITY;
4763 put_unaligned_le32(sizeof(*capability),
4764 &request.data.report_device_capability.buffer_length);
4765
4766 rc = pqi_map_single(ctrl_info->pci_dev,
4767 &request.data.report_device_capability.sg_descriptor,
4768 capability, sizeof(*capability),
4769 DMA_FROM_DEVICE);
4770 if (rc)
4771 goto out;
4772
4773 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request, &response);
4774
4775 pqi_pci_unmap(ctrl_info->pci_dev,
4776 &request.data.report_device_capability.sg_descriptor, 1,
4777 DMA_FROM_DEVICE);
4778
4779 if (rc)
4780 goto out;
4781
4782 if (response.status != PQI_GENERAL_ADMIN_STATUS_SUCCESS) {
4783 rc = -EIO;
4784 goto out;
4785 }
4786
4787 ctrl_info->max_inbound_queues =
4788 get_unaligned_le16(&capability->max_inbound_queues);
4789 ctrl_info->max_elements_per_iq =
4790 get_unaligned_le16(&capability->max_elements_per_iq);
4791 ctrl_info->max_iq_element_length =
4792 get_unaligned_le16(&capability->max_iq_element_length)
4793 * 16;
4794 ctrl_info->max_outbound_queues =
4795 get_unaligned_le16(&capability->max_outbound_queues);
4796 ctrl_info->max_elements_per_oq =
4797 get_unaligned_le16(&capability->max_elements_per_oq);
4798 ctrl_info->max_oq_element_length =
4799 get_unaligned_le16(&capability->max_oq_element_length)
4800 * 16;
4801
4802 sop_iu_layer_descriptor =
4803 &capability->iu_layer_descriptors[PQI_PROTOCOL_SOP];
4804
4805 ctrl_info->max_inbound_iu_length_per_firmware =
4806 get_unaligned_le16(
4807 &sop_iu_layer_descriptor->max_inbound_iu_length);
4808 ctrl_info->inbound_spanning_supported =
4809 sop_iu_layer_descriptor->inbound_spanning_supported;
4810 ctrl_info->outbound_spanning_supported =
4811 sop_iu_layer_descriptor->outbound_spanning_supported;
4812
4813out:
4814 kfree(capability);
4815
4816 return rc;
4817}
4818
4819static int pqi_validate_device_capability(struct pqi_ctrl_info *ctrl_info)
4820{
4821 if (ctrl_info->max_iq_element_length <
4822 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) {
4823 dev_err(&ctrl_info->pci_dev->dev,
4824 "max. inbound queue element length of %d is less than the required length of %d\n",
4825 ctrl_info->max_iq_element_length,
4826 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
4827 return -EINVAL;
4828 }
4829
4830 if (ctrl_info->max_oq_element_length <
4831 PQI_OPERATIONAL_OQ_ELEMENT_LENGTH) {
4832 dev_err(&ctrl_info->pci_dev->dev,
4833 "max. outbound queue element length of %d is less than the required length of %d\n",
4834 ctrl_info->max_oq_element_length,
4835 PQI_OPERATIONAL_OQ_ELEMENT_LENGTH);
4836 return -EINVAL;
4837 }
4838
4839 if (ctrl_info->max_inbound_iu_length_per_firmware <
4840 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) {
4841 dev_err(&ctrl_info->pci_dev->dev,
4842 "max. inbound IU length of %u is less than the min. required length of %d\n",
4843 ctrl_info->max_inbound_iu_length_per_firmware,
4844 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
4845 return -EINVAL;
4846 }
4847
4848 if (!ctrl_info->inbound_spanning_supported) {
4849 dev_err(&ctrl_info->pci_dev->dev,
4850 "the controller does not support inbound spanning\n");
4851 return -EINVAL;
4852 }
4853
4854 if (ctrl_info->outbound_spanning_supported) {
4855 dev_err(&ctrl_info->pci_dev->dev,
4856 "the controller supports outbound spanning but this driver does not\n");
4857 return -EINVAL;
4858 }
4859
4860 return 0;
4861}
4862
4863static int pqi_create_event_queue(struct pqi_ctrl_info *ctrl_info)
4864{
4865 int rc;
4866 struct pqi_event_queue *event_queue;
4867 struct pqi_general_admin_request request;
4868 struct pqi_general_admin_response response;
4869
4870 event_queue = &ctrl_info->event_queue;
4871
4872 /*
4873 * Create OQ (Outbound Queue - device to host queue) to dedicate
4874 * to events.
4875 */
4876 memset(&request, 0, sizeof(request));
4877 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4878 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4879 &request.header.iu_length);
4880 request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_OQ;
4881 put_unaligned_le16(event_queue->oq_id,
4882 &request.data.create_operational_oq.queue_id);
4883 put_unaligned_le64((u64)event_queue->oq_element_array_bus_addr,
4884 &request.data.create_operational_oq.element_array_addr);
4885 put_unaligned_le64((u64)event_queue->oq_pi_bus_addr,
4886 &request.data.create_operational_oq.pi_addr);
4887 put_unaligned_le16(PQI_NUM_EVENT_QUEUE_ELEMENTS,
4888 &request.data.create_operational_oq.num_elements);
4889 put_unaligned_le16(PQI_EVENT_OQ_ELEMENT_LENGTH / 16,
4890 &request.data.create_operational_oq.element_length);
4891 request.data.create_operational_oq.queue_protocol = PQI_PROTOCOL_SOP;
4892 put_unaligned_le16(event_queue->int_msg_num,
4893 &request.data.create_operational_oq.int_msg_num);
4894
4895 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4896 &response);
4897 if (rc)
4898 return rc;
4899
4900 event_queue->oq_ci = ctrl_info->iomem_base +
4901 PQI_DEVICE_REGISTERS_OFFSET +
4902 get_unaligned_le64(
4903 &response.data.create_operational_oq.oq_ci_offset);
4904
4905 return 0;
4906}
4907
4908static int pqi_create_queue_group(struct pqi_ctrl_info *ctrl_info,
4909 unsigned int group_number)
4910{
4911 int rc;
4912 struct pqi_queue_group *queue_group;
4913 struct pqi_general_admin_request request;
4914 struct pqi_general_admin_response response;
4915
4916 queue_group = &ctrl_info->queue_groups[group_number];
4917
4918 /*
4919 * Create IQ (Inbound Queue - host to device queue) for
4920 * RAID path.
4921 */
4922 memset(&request, 0, sizeof(request));
4923 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4924 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4925 &request.header.iu_length);
4926 request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_IQ;
4927 put_unaligned_le16(queue_group->iq_id[RAID_PATH],
4928 &request.data.create_operational_iq.queue_id);
4929 put_unaligned_le64(
4930 (u64)queue_group->iq_element_array_bus_addr[RAID_PATH],
4931 &request.data.create_operational_iq.element_array_addr);
4932 put_unaligned_le64((u64)queue_group->iq_ci_bus_addr[RAID_PATH],
4933 &request.data.create_operational_iq.ci_addr);
4934 put_unaligned_le16(ctrl_info->num_elements_per_iq,
4935 &request.data.create_operational_iq.num_elements);
4936 put_unaligned_le16(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH / 16,
4937 &request.data.create_operational_iq.element_length);
4938 request.data.create_operational_iq.queue_protocol = PQI_PROTOCOL_SOP;
4939
4940 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4941 &response);
4942 if (rc) {
4943 dev_err(&ctrl_info->pci_dev->dev,
4944 "error creating inbound RAID queue\n");
4945 return rc;
4946 }
4947
4948 queue_group->iq_pi[RAID_PATH] = ctrl_info->iomem_base +
4949 PQI_DEVICE_REGISTERS_OFFSET +
4950 get_unaligned_le64(
4951 &response.data.create_operational_iq.iq_pi_offset);
4952
4953 /*
4954 * Create IQ (Inbound Queue - host to device queue) for
4955 * Advanced I/O (AIO) path.
4956 */
4957 memset(&request, 0, sizeof(request));
4958 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4959 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4960 &request.header.iu_length);
4961 request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_IQ;
4962 put_unaligned_le16(queue_group->iq_id[AIO_PATH],
4963 &request.data.create_operational_iq.queue_id);
4964 put_unaligned_le64((u64)queue_group->
4965 iq_element_array_bus_addr[AIO_PATH],
4966 &request.data.create_operational_iq.element_array_addr);
4967 put_unaligned_le64((u64)queue_group->iq_ci_bus_addr[AIO_PATH],
4968 &request.data.create_operational_iq.ci_addr);
4969 put_unaligned_le16(ctrl_info->num_elements_per_iq,
4970 &request.data.create_operational_iq.num_elements);
4971 put_unaligned_le16(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH / 16,
4972 &request.data.create_operational_iq.element_length);
4973 request.data.create_operational_iq.queue_protocol = PQI_PROTOCOL_SOP;
4974
4975 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4976 &response);
4977 if (rc) {
4978 dev_err(&ctrl_info->pci_dev->dev,
4979 "error creating inbound AIO queue\n");
4980 return rc;
4981 }
4982
4983 queue_group->iq_pi[AIO_PATH] = ctrl_info->iomem_base +
4984 PQI_DEVICE_REGISTERS_OFFSET +
4985 get_unaligned_le64(
4986 &response.data.create_operational_iq.iq_pi_offset);
4987
4988 /*
4989 * Designate the 2nd IQ as the AIO path. By default, all IQs are
4990 * assumed to be for RAID path I/O unless we change the queue's
4991 * property.
4992 */
4993 memset(&request, 0, sizeof(request));
4994 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4995 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4996 &request.header.iu_length);
4997 request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CHANGE_IQ_PROPERTY;
4998 put_unaligned_le16(queue_group->iq_id[AIO_PATH],
4999 &request.data.change_operational_iq_properties.queue_id);
5000 put_unaligned_le32(PQI_IQ_PROPERTY_IS_AIO_QUEUE,
5001 &request.data.change_operational_iq_properties.vendor_specific);
5002
5003 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
5004 &response);
5005 if (rc) {
5006 dev_err(&ctrl_info->pci_dev->dev,
5007 "error changing queue property\n");
5008 return rc;
5009 }
5010
5011 /*
5012 * Create OQ (Outbound Queue - device to host queue).
5013 */
5014 memset(&request, 0, sizeof(request));
5015 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
5016 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
5017 &request.header.iu_length);
5018 request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_OQ;
5019 put_unaligned_le16(queue_group->oq_id,
5020 &request.data.create_operational_oq.queue_id);
5021 put_unaligned_le64((u64)queue_group->oq_element_array_bus_addr,
5022 &request.data.create_operational_oq.element_array_addr);
5023 put_unaligned_le64((u64)queue_group->oq_pi_bus_addr,
5024 &request.data.create_operational_oq.pi_addr);
5025 put_unaligned_le16(ctrl_info->num_elements_per_oq,
5026 &request.data.create_operational_oq.num_elements);
5027 put_unaligned_le16(PQI_OPERATIONAL_OQ_ELEMENT_LENGTH / 16,
5028 &request.data.create_operational_oq.element_length);
5029 request.data.create_operational_oq.queue_protocol = PQI_PROTOCOL_SOP;
5030 put_unaligned_le16(queue_group->int_msg_num,
5031 &request.data.create_operational_oq.int_msg_num);
5032
5033 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
5034 &response);
5035 if (rc) {
5036 dev_err(&ctrl_info->pci_dev->dev,
5037 "error creating outbound queue\n");
5038 return rc;
5039 }
5040
5041 queue_group->oq_ci = ctrl_info->iomem_base +
5042 PQI_DEVICE_REGISTERS_OFFSET +
5043 get_unaligned_le64(
5044 &response.data.create_operational_oq.oq_ci_offset);
5045
5046 return 0;
5047}
5048
5049static int pqi_create_queues(struct pqi_ctrl_info *ctrl_info)
5050{
5051 int rc;
5052 unsigned int i;
5053
5054 rc = pqi_create_event_queue(ctrl_info);
5055 if (rc) {
5056 dev_err(&ctrl_info->pci_dev->dev,
5057 "error creating event queue\n");
5058 return rc;
5059 }
5060
5061 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
5062 rc = pqi_create_queue_group(ctrl_info, i);
5063 if (rc) {
5064 dev_err(&ctrl_info->pci_dev->dev,
5065 "error creating queue group number %u/%u\n",
5066 i, ctrl_info->num_queue_groups);
5067 return rc;
5068 }
5069 }
5070
5071 return 0;
5072}
5073
5074#define PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH \
5075 struct_size_t(struct pqi_event_config, descriptors, PQI_MAX_EVENT_DESCRIPTORS)
5076
5077static int pqi_configure_events(struct pqi_ctrl_info *ctrl_info,
5078 bool enable_events)
5079{
5080 int rc;
5081 unsigned int i;
5082 struct pqi_event_config *event_config;
5083 struct pqi_event_descriptor *event_descriptor;
5084 struct pqi_general_management_request request;
5085
5086 event_config = kmalloc(PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
5087 GFP_KERNEL);
5088 if (!event_config)
5089 return -ENOMEM;
5090
5091 memset(&request, 0, sizeof(request));
5092
5093 request.header.iu_type = PQI_REQUEST_IU_REPORT_VENDOR_EVENT_CONFIG;
5094 put_unaligned_le16(offsetof(struct pqi_general_management_request,
5095 data.report_event_configuration.sg_descriptors[1]) -
5096 PQI_REQUEST_HEADER_LENGTH, &request.header.iu_length);
5097 put_unaligned_le32(PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
5098 &request.data.report_event_configuration.buffer_length);
5099
5100 rc = pqi_map_single(ctrl_info->pci_dev,
5101 request.data.report_event_configuration.sg_descriptors,
5102 event_config, PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
5103 DMA_FROM_DEVICE);
5104 if (rc)
5105 goto out;
5106
5107 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);
5108
5109 pqi_pci_unmap(ctrl_info->pci_dev,
5110 request.data.report_event_configuration.sg_descriptors, 1,
5111 DMA_FROM_DEVICE);
5112
5113 if (rc)
5114 goto out;
5115
5116 for (i = 0; i < event_config->num_event_descriptors; i++) {
5117 event_descriptor = &event_config->descriptors[i];
5118 if (enable_events &&
5119 pqi_is_supported_event(event_descriptor->event_type))
5120 put_unaligned_le16(ctrl_info->event_queue.oq_id,
5121 &event_descriptor->oq_id);
5122 else
5123 put_unaligned_le16(0, &event_descriptor->oq_id);
5124 }
5125
5126 memset(&request, 0, sizeof(request));
5127
5128 request.header.iu_type = PQI_REQUEST_IU_SET_VENDOR_EVENT_CONFIG;
5129 put_unaligned_le16(offsetof(struct pqi_general_management_request,
5130 data.report_event_configuration.sg_descriptors[1]) -
5131 PQI_REQUEST_HEADER_LENGTH, &request.header.iu_length);
5132 put_unaligned_le32(PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
5133 &request.data.report_event_configuration.buffer_length);
5134
5135 rc = pqi_map_single(ctrl_info->pci_dev,
5136 request.data.report_event_configuration.sg_descriptors,
5137 event_config, PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
5138 DMA_TO_DEVICE);
5139 if (rc)
5140 goto out;
5141
5142 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);
5143
5144 pqi_pci_unmap(ctrl_info->pci_dev,
5145 request.data.report_event_configuration.sg_descriptors, 1,
5146 DMA_TO_DEVICE);
5147
5148out:
5149 kfree(event_config);
5150
5151 return rc;
5152}
5153
5154static inline int pqi_enable_events(struct pqi_ctrl_info *ctrl_info)
5155{
5156 return pqi_configure_events(ctrl_info, true);
5157}
5158
5159static void pqi_free_all_io_requests(struct pqi_ctrl_info *ctrl_info)
5160{
5161 unsigned int i;
5162 struct device *dev;
5163 size_t sg_chain_buffer_length;
5164 struct pqi_io_request *io_request;
5165
5166 if (!ctrl_info->io_request_pool)
5167 return;
5168
5169 dev = &ctrl_info->pci_dev->dev;
5170 sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length;
5171 io_request = ctrl_info->io_request_pool;
5172
5173 for (i = 0; i < ctrl_info->max_io_slots; i++) {
5174 kfree(io_request->iu);
5175 if (!io_request->sg_chain_buffer)
5176 break;
5177 dma_free_coherent(dev, sg_chain_buffer_length,
5178 io_request->sg_chain_buffer,
5179 io_request->sg_chain_buffer_dma_handle);
5180 io_request++;
5181 }
5182
5183 kfree(ctrl_info->io_request_pool);
5184 ctrl_info->io_request_pool = NULL;
5185}
5186
5187static inline int pqi_alloc_error_buffer(struct pqi_ctrl_info *ctrl_info)
5188{
5189 ctrl_info->error_buffer = dma_alloc_coherent(&ctrl_info->pci_dev->dev,
5190 ctrl_info->error_buffer_length,
5191 &ctrl_info->error_buffer_dma_handle,
5192 GFP_KERNEL);
5193 if (!ctrl_info->error_buffer)
5194 return -ENOMEM;
5195
5196 return 0;
5197}
5198
5199static int pqi_alloc_io_resources(struct pqi_ctrl_info *ctrl_info)
5200{
5201 unsigned int i;
5202 void *sg_chain_buffer;
5203 size_t sg_chain_buffer_length;
5204 dma_addr_t sg_chain_buffer_dma_handle;
5205 struct device *dev;
5206 struct pqi_io_request *io_request;
5207
5208 ctrl_info->io_request_pool = kzalloc_objs(ctrl_info->io_request_pool[0],
5209 ctrl_info->max_io_slots);
5210
5211 if (!ctrl_info->io_request_pool) {
5212 dev_err(&ctrl_info->pci_dev->dev,
5213 "failed to allocate I/O request pool\n");
5214 goto error;
5215 }
5216
5217 dev = &ctrl_info->pci_dev->dev;
5218 sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length;
5219 io_request = ctrl_info->io_request_pool;
5220
5221 for (i = 0; i < ctrl_info->max_io_slots; i++) {
5222 io_request->iu = kmalloc(ctrl_info->max_inbound_iu_length, GFP_KERNEL);
5223
5224 if (!io_request->iu) {
5225 dev_err(&ctrl_info->pci_dev->dev,
5226 "failed to allocate IU buffers\n");
5227 goto error;
5228 }
5229
5230 sg_chain_buffer = dma_alloc_coherent(dev,
5231 sg_chain_buffer_length, &sg_chain_buffer_dma_handle,
5232 GFP_KERNEL);
5233
5234 if (!sg_chain_buffer) {
5235 dev_err(&ctrl_info->pci_dev->dev,
5236 "failed to allocate PQI scatter-gather chain buffers\n");
5237 goto error;
5238 }
5239
5240 io_request->index = i;
5241 io_request->sg_chain_buffer = sg_chain_buffer;
5242 io_request->sg_chain_buffer_dma_handle = sg_chain_buffer_dma_handle;
5243 io_request++;
5244 }
5245
5246 return 0;
5247
5248error:
5249 pqi_free_all_io_requests(ctrl_info);
5250
5251 return -ENOMEM;
5252}
5253
5254/*
5255 * Calculate required resources that are sized based on max. outstanding
5256 * requests and max. transfer size.
5257 */
5258
5259static void pqi_calculate_io_resources(struct pqi_ctrl_info *ctrl_info)
5260{
5261 u32 max_transfer_size;
5262 u32 max_sg_entries;
5263
5264 ctrl_info->scsi_ml_can_queue =
5265 ctrl_info->max_outstanding_requests - PQI_RESERVED_IO_SLOTS;
5266 ctrl_info->max_io_slots = ctrl_info->max_outstanding_requests;
5267
5268 ctrl_info->error_buffer_length =
5269 ctrl_info->max_io_slots * PQI_ERROR_BUFFER_ELEMENT_LENGTH;
5270
5271 if (is_kdump_kernel())
5272 max_transfer_size = min(ctrl_info->max_transfer_size,
5273 PQI_MAX_TRANSFER_SIZE_KDUMP);
5274 else
5275 max_transfer_size = min(ctrl_info->max_transfer_size,
5276 PQI_MAX_TRANSFER_SIZE);
5277
5278 max_sg_entries = max_transfer_size / PAGE_SIZE;
5279
5280 /* +1 to cover when the buffer is not page-aligned. */
5281 max_sg_entries++;
5282
5283 max_sg_entries = min(ctrl_info->max_sg_entries, max_sg_entries);
5284
5285 max_transfer_size = (max_sg_entries - 1) * PAGE_SIZE;
5286
5287 ctrl_info->sg_chain_buffer_length =
5288 (max_sg_entries * sizeof(struct pqi_sg_descriptor)) +
5289 PQI_EXTRA_SGL_MEMORY;
5290 ctrl_info->sg_tablesize = max_sg_entries;
5291 ctrl_info->max_sectors = max_transfer_size / 512;
5292}
5293
5294static void pqi_calculate_queue_resources(struct pqi_ctrl_info *ctrl_info)
5295{
5296 int num_queue_groups;
5297 u16 num_elements_per_iq;
5298 u16 num_elements_per_oq;
5299
5300 if (is_kdump_kernel()) {
5301 num_queue_groups = 1;
5302 } else {
5303 int max_queue_groups;
5304
5305 max_queue_groups = min(ctrl_info->max_inbound_queues / 2,
5306 ctrl_info->max_outbound_queues - 1);
5307 max_queue_groups = min(max_queue_groups, PQI_MAX_QUEUE_GROUPS);
5308
5309 num_queue_groups =
5310 blk_mq_num_online_queues(ctrl_info->max_msix_vectors);
5311 num_queue_groups = min(num_queue_groups, max_queue_groups);
5312 }
5313
5314 ctrl_info->num_queue_groups = num_queue_groups;
5315
5316 /*
5317 * Make sure that the max. inbound IU length is an even multiple
5318 * of our inbound element length.
5319 */
5320 ctrl_info->max_inbound_iu_length =
5321 (ctrl_info->max_inbound_iu_length_per_firmware /
5322 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) *
5323 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH;
5324
5325 num_elements_per_iq =
5326 (ctrl_info->max_inbound_iu_length /
5327 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
5328
5329 /* Add one because one element in each queue is unusable. */
5330 num_elements_per_iq++;
5331
5332 num_elements_per_iq = min(num_elements_per_iq,
5333 ctrl_info->max_elements_per_iq);
5334
5335 num_elements_per_oq = ((num_elements_per_iq - 1) * 2) + 1;
5336 num_elements_per_oq = min(num_elements_per_oq,
5337 ctrl_info->max_elements_per_oq);
5338
5339 ctrl_info->num_elements_per_iq = num_elements_per_iq;
5340 ctrl_info->num_elements_per_oq = num_elements_per_oq;
5341
5342 ctrl_info->max_sg_per_iu =
5343 ((ctrl_info->max_inbound_iu_length -
5344 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) /
5345 sizeof(struct pqi_sg_descriptor)) +
5346 PQI_MAX_EMBEDDED_SG_DESCRIPTORS;
5347
5348 ctrl_info->max_sg_per_r56_iu =
5349 ((ctrl_info->max_inbound_iu_length -
5350 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) /
5351 sizeof(struct pqi_sg_descriptor)) +
5352 PQI_MAX_EMBEDDED_R56_SG_DESCRIPTORS;
5353}
5354
5355static inline void pqi_set_sg_descriptor(struct pqi_sg_descriptor *sg_descriptor,
5356 struct scatterlist *sg)
5357{
5358 u64 address = (u64)sg_dma_address(sg);
5359 unsigned int length = sg_dma_len(sg);
5360
5361 put_unaligned_le64(address, &sg_descriptor->address);
5362 put_unaligned_le32(length, &sg_descriptor->length);
5363 put_unaligned_le32(0, &sg_descriptor->flags);
5364}
5365
5366static unsigned int pqi_build_sg_list(struct pqi_sg_descriptor *sg_descriptor,
5367 struct scatterlist *sg, int sg_count, struct pqi_io_request *io_request,
5368 int max_sg_per_iu, bool *chained)
5369{
5370 int i;
5371 unsigned int num_sg_in_iu;
5372
5373 *chained = false;
5374 i = 0;
5375 num_sg_in_iu = 0;
5376 max_sg_per_iu--; /* Subtract 1 to leave room for chain marker. */
5377
5378 while (1) {
5379 pqi_set_sg_descriptor(sg_descriptor, sg);
5380 if (!*chained)
5381 num_sg_in_iu++;
5382 i++;
5383 if (i == sg_count)
5384 break;
5385 sg_descriptor++;
5386 if (i == max_sg_per_iu) {
5387 put_unaligned_le64((u64)io_request->sg_chain_buffer_dma_handle,
5388 &sg_descriptor->address);
5389 put_unaligned_le32((sg_count - num_sg_in_iu) * sizeof(*sg_descriptor),
5390 &sg_descriptor->length);
5391 put_unaligned_le32(CISS_SG_CHAIN, &sg_descriptor->flags);
5392 *chained = true;
5393 num_sg_in_iu++;
5394 sg_descriptor = io_request->sg_chain_buffer;
5395 }
5396 sg = sg_next(sg);
5397 }
5398
5399 put_unaligned_le32(CISS_SG_LAST, &sg_descriptor->flags);
5400
5401 return num_sg_in_iu;
5402}
5403
5404static int pqi_build_raid_sg_list(struct pqi_ctrl_info *ctrl_info,
5405 struct pqi_raid_path_request *request, struct scsi_cmnd *scmd,
5406 struct pqi_io_request *io_request)
5407{
5408 u16 iu_length;
5409 int sg_count;
5410 bool chained;
5411 unsigned int num_sg_in_iu;
5412 struct scatterlist *sg;
5413 struct pqi_sg_descriptor *sg_descriptor;
5414
5415 sg_count = scsi_dma_map(scmd);
5416 if (sg_count < 0)
5417 return sg_count;
5418
5419 iu_length = offsetof(struct pqi_raid_path_request, sg_descriptors) -
5420 PQI_REQUEST_HEADER_LENGTH;
5421
5422 if (sg_count == 0)
5423 goto out;
5424
5425 sg = scsi_sglist(scmd);
5426 sg_descriptor = request->sg_descriptors;
5427
5428 num_sg_in_iu = pqi_build_sg_list(sg_descriptor, sg, sg_count, io_request,
5429 ctrl_info->max_sg_per_iu, &chained);
5430
5431 request->partial = chained;
5432 iu_length += num_sg_in_iu * sizeof(*sg_descriptor);
5433
5434out:
5435 put_unaligned_le16(iu_length, &request->header.iu_length);
5436
5437 return 0;
5438}
5439
5440static int pqi_build_aio_r1_sg_list(struct pqi_ctrl_info *ctrl_info,
5441 struct pqi_aio_r1_path_request *request, struct scsi_cmnd *scmd,
5442 struct pqi_io_request *io_request)
5443{
5444 u16 iu_length;
5445 int sg_count;
5446 bool chained;
5447 unsigned int num_sg_in_iu;
5448 struct scatterlist *sg;
5449 struct pqi_sg_descriptor *sg_descriptor;
5450
5451 sg_count = scsi_dma_map(scmd);
5452 if (sg_count < 0)
5453 return sg_count;
5454
5455 iu_length = offsetof(struct pqi_aio_r1_path_request, sg_descriptors) -
5456 PQI_REQUEST_HEADER_LENGTH;
5457 num_sg_in_iu = 0;
5458
5459 if (sg_count == 0)
5460 goto out;
5461
5462 sg = scsi_sglist(scmd);
5463 sg_descriptor = request->sg_descriptors;
5464
5465 num_sg_in_iu = pqi_build_sg_list(sg_descriptor, sg, sg_count, io_request,
5466 ctrl_info->max_sg_per_iu, &chained);
5467
5468 request->partial = chained;
5469 iu_length += num_sg_in_iu * sizeof(*sg_descriptor);
5470
5471out:
5472 put_unaligned_le16(iu_length, &request->header.iu_length);
5473 request->num_sg_descriptors = num_sg_in_iu;
5474
5475 return 0;
5476}
5477
5478static int pqi_build_aio_r56_sg_list(struct pqi_ctrl_info *ctrl_info,
5479 struct pqi_aio_r56_path_request *request, struct scsi_cmnd *scmd,
5480 struct pqi_io_request *io_request)
5481{
5482 u16 iu_length;
5483 int sg_count;
5484 bool chained;
5485 unsigned int num_sg_in_iu;
5486 struct scatterlist *sg;
5487 struct pqi_sg_descriptor *sg_descriptor;
5488
5489 sg_count = scsi_dma_map(scmd);
5490 if (sg_count < 0)
5491 return sg_count;
5492
5493 iu_length = offsetof(struct pqi_aio_r56_path_request, sg_descriptors) -
5494 PQI_REQUEST_HEADER_LENGTH;
5495 num_sg_in_iu = 0;
5496
5497 if (sg_count != 0) {
5498 sg = scsi_sglist(scmd);
5499 sg_descriptor = request->sg_descriptors;
5500
5501 num_sg_in_iu = pqi_build_sg_list(sg_descriptor, sg, sg_count, io_request,
5502 ctrl_info->max_sg_per_r56_iu, &chained);
5503
5504 request->partial = chained;
5505 iu_length += num_sg_in_iu * sizeof(*sg_descriptor);
5506 }
5507
5508 put_unaligned_le16(iu_length, &request->header.iu_length);
5509 request->num_sg_descriptors = num_sg_in_iu;
5510
5511 return 0;
5512}
5513
5514static int pqi_build_aio_sg_list(struct pqi_ctrl_info *ctrl_info,
5515 struct pqi_aio_path_request *request, struct scsi_cmnd *scmd,
5516 struct pqi_io_request *io_request)
5517{
5518 u16 iu_length;
5519 int sg_count;
5520 bool chained;
5521 unsigned int num_sg_in_iu;
5522 struct scatterlist *sg;
5523 struct pqi_sg_descriptor *sg_descriptor;
5524
5525 sg_count = scsi_dma_map(scmd);
5526 if (sg_count < 0)
5527 return sg_count;
5528
5529 iu_length = offsetof(struct pqi_aio_path_request, sg_descriptors) -
5530 PQI_REQUEST_HEADER_LENGTH;
5531 num_sg_in_iu = 0;
5532
5533 if (sg_count == 0)
5534 goto out;
5535
5536 sg = scsi_sglist(scmd);
5537 sg_descriptor = request->sg_descriptors;
5538
5539 num_sg_in_iu = pqi_build_sg_list(sg_descriptor, sg, sg_count, io_request,
5540 ctrl_info->max_sg_per_iu, &chained);
5541
5542 request->partial = chained;
5543 iu_length += num_sg_in_iu * sizeof(*sg_descriptor);
5544
5545out:
5546 put_unaligned_le16(iu_length, &request->header.iu_length);
5547 request->num_sg_descriptors = num_sg_in_iu;
5548
5549 return 0;
5550}
5551
5552static void pqi_raid_io_complete(struct pqi_io_request *io_request,
5553 void *context)
5554{
5555 struct scsi_cmnd *scmd;
5556
5557 scmd = io_request->scmd;
5558 pqi_free_io_request(io_request);
5559 scsi_dma_unmap(scmd);
5560 pqi_scsi_done(scmd);
5561}
5562
5563/*
5564 * Adjust the timeout value for physical devices sent to the firmware
5565 * by subtracting 3 seconds for timeouts greater than or equal to 8 seconds.
5566 *
5567 * This provides the firmware with additional time to attempt early recovery
5568 * before the OS-level timeout occurs.
5569 */
5570#define ADJUST_SECS_TIMEOUT_VALUE(tv) (((tv) >= 8) ? ((tv) - 3) : (tv))
5571
5572static int pqi_raid_submit_io(struct pqi_ctrl_info *ctrl_info,
5573 struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
5574 struct pqi_queue_group *queue_group, bool io_high_prio)
5575{
5576 int rc;
5577 u32 timeout;
5578 size_t cdb_length;
5579 struct pqi_io_request *io_request;
5580 struct pqi_raid_path_request *request;
5581 struct request *rq;
5582
5583 io_request = pqi_alloc_io_request(ctrl_info, scmd);
5584 if (!io_request)
5585 return SCSI_MLQUEUE_HOST_BUSY;
5586
5587 io_request->io_complete_callback = pqi_raid_io_complete;
5588 io_request->scmd = scmd;
5589
5590 request = io_request->iu;
5591 memset(request, 0, offsetof(struct pqi_raid_path_request, sg_descriptors));
5592
5593 request->header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO;
5594 put_unaligned_le32(scsi_bufflen(scmd), &request->buffer_length);
5595 request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
5596 request->command_priority = io_high_prio;
5597 put_unaligned_le16(io_request->index, &request->request_id);
5598 request->error_index = request->request_id;
5599 memcpy(request->lun_number, device->scsi3addr, sizeof(request->lun_number));
5600 request->ml_device_lun_number = (u8)scmd->device->lun;
5601
5602 cdb_length = min_t(size_t, scmd->cmd_len, sizeof(request->cdb));
5603 memcpy(request->cdb, scmd->cmnd, cdb_length);
5604
5605 switch (cdb_length) {
5606 case 6:
5607 case 10:
5608 case 12:
5609 case 16:
5610 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_0;
5611 break;
5612 case 20:
5613 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_4;
5614 break;
5615 case 24:
5616 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_8;
5617 break;
5618 case 28:
5619 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_12;
5620 break;
5621 case 32:
5622 default:
5623 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_16;
5624 break;
5625 }
5626
5627 switch (scmd->sc_data_direction) {
5628 case DMA_FROM_DEVICE:
5629 request->data_direction = SOP_READ_FLAG;
5630 break;
5631 case DMA_TO_DEVICE:
5632 request->data_direction = SOP_WRITE_FLAG;
5633 break;
5634 case DMA_NONE:
5635 request->data_direction = SOP_NO_DIRECTION_FLAG;
5636 break;
5637 case DMA_BIDIRECTIONAL:
5638 request->data_direction = SOP_BIDIRECTIONAL;
5639 break;
5640 default:
5641 dev_err(&ctrl_info->pci_dev->dev,
5642 "unknown data direction: %d\n",
5643 scmd->sc_data_direction);
5644 break;
5645 }
5646
5647 rc = pqi_build_raid_sg_list(ctrl_info, request, scmd, io_request);
5648 if (rc) {
5649 pqi_free_io_request(io_request);
5650 return SCSI_MLQUEUE_HOST_BUSY;
5651 }
5652
5653 if (device->is_physical_device) {
5654 rq = scsi_cmd_to_rq(scmd);
5655 timeout = rq->timeout / HZ;
5656 put_unaligned_le32(ADJUST_SECS_TIMEOUT_VALUE(timeout), &request->timeout);
5657 }
5658
5659 pqi_start_io(ctrl_info, queue_group, RAID_PATH, io_request);
5660
5661 return 0;
5662}
5663
5664static inline int pqi_raid_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,
5665 struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
5666 struct pqi_queue_group *queue_group)
5667{
5668 bool io_high_prio;
5669
5670 io_high_prio = pqi_is_io_high_priority(device, scmd);
5671
5672 return pqi_raid_submit_io(ctrl_info, device, scmd, queue_group, io_high_prio);
5673}
5674
5675static bool pqi_raid_bypass_retry_needed(struct pqi_io_request *io_request)
5676{
5677 struct scsi_cmnd *scmd;
5678 struct pqi_scsi_dev *device;
5679 struct pqi_ctrl_info *ctrl_info;
5680
5681 if (!io_request->raid_bypass)
5682 return false;
5683
5684 scmd = io_request->scmd;
5685 if ((scmd->result & 0xff) == SAM_STAT_GOOD)
5686 return false;
5687 if (host_byte(scmd->result) == DID_NO_CONNECT)
5688 return false;
5689
5690 device = scmd->device->hostdata;
5691 if (pqi_device_offline(device) || pqi_device_in_remove(device))
5692 return false;
5693
5694 ctrl_info = shost_to_hba(scmd->device->host);
5695 if (pqi_ctrl_offline(ctrl_info))
5696 return false;
5697
5698 return true;
5699}
5700
5701static void pqi_aio_io_complete(struct pqi_io_request *io_request,
5702 void *context)
5703{
5704 struct scsi_cmnd *scmd;
5705
5706 scmd = io_request->scmd;
5707 scsi_dma_unmap(scmd);
5708 if (io_request->status == -EAGAIN || pqi_raid_bypass_retry_needed(io_request)) {
5709 set_host_byte(scmd, DID_IMM_RETRY);
5710 pqi_cmd_priv(scmd)->this_residual++;
5711 }
5712
5713 pqi_free_io_request(io_request);
5714 pqi_scsi_done(scmd);
5715}
5716
5717static inline int pqi_aio_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,
5718 struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
5719 struct pqi_queue_group *queue_group)
5720{
5721 bool io_high_prio;
5722
5723 io_high_prio = pqi_is_io_high_priority(device, scmd);
5724
5725 return pqi_aio_submit_io(ctrl_info, scmd, device->aio_handle,
5726 scmd->cmnd, scmd->cmd_len, queue_group, NULL,
5727 false, io_high_prio);
5728}
5729
5730static int pqi_aio_submit_io(struct pqi_ctrl_info *ctrl_info,
5731 struct scsi_cmnd *scmd, u32 aio_handle, u8 *cdb,
5732 unsigned int cdb_length, struct pqi_queue_group *queue_group,
5733 struct pqi_encryption_info *encryption_info, bool raid_bypass,
5734 bool io_high_prio)
5735{
5736 int rc;
5737 struct pqi_io_request *io_request;
5738 struct pqi_aio_path_request *request;
5739
5740 io_request = pqi_alloc_io_request(ctrl_info, scmd);
5741 if (!io_request)
5742 return SCSI_MLQUEUE_HOST_BUSY;
5743
5744 io_request->io_complete_callback = pqi_aio_io_complete;
5745 io_request->scmd = scmd;
5746 io_request->raid_bypass = raid_bypass;
5747
5748 request = io_request->iu;
5749 memset(request, 0, offsetof(struct pqi_aio_path_request, sg_descriptors));
5750
5751 request->header.iu_type = PQI_REQUEST_IU_AIO_PATH_IO;
5752 put_unaligned_le32(aio_handle, &request->nexus_id);
5753 put_unaligned_le32(scsi_bufflen(scmd), &request->buffer_length);
5754 request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
5755 request->command_priority = io_high_prio;
5756 put_unaligned_le16(io_request->index, &request->request_id);
5757 request->error_index = request->request_id;
5758 if (!raid_bypass && ctrl_info->multi_lun_device_supported)
5759 put_unaligned_le64(scmd->device->lun << 8, &request->lun_number);
5760 if (cdb_length > sizeof(request->cdb))
5761 cdb_length = sizeof(request->cdb);
5762 request->cdb_length = cdb_length;
5763 memcpy(request->cdb, cdb, cdb_length);
5764
5765 switch (scmd->sc_data_direction) {
5766 case DMA_TO_DEVICE:
5767 request->data_direction = SOP_READ_FLAG;
5768 break;
5769 case DMA_FROM_DEVICE:
5770 request->data_direction = SOP_WRITE_FLAG;
5771 break;
5772 case DMA_NONE:
5773 request->data_direction = SOP_NO_DIRECTION_FLAG;
5774 break;
5775 case DMA_BIDIRECTIONAL:
5776 request->data_direction = SOP_BIDIRECTIONAL;
5777 break;
5778 default:
5779 dev_err(&ctrl_info->pci_dev->dev,
5780 "unknown data direction: %d\n",
5781 scmd->sc_data_direction);
5782 break;
5783 }
5784
5785 if (encryption_info) {
5786 request->encryption_enable = true;
5787 put_unaligned_le16(encryption_info->data_encryption_key_index,
5788 &request->data_encryption_key_index);
5789 put_unaligned_le32(encryption_info->encrypt_tweak_lower,
5790 &request->encrypt_tweak_lower);
5791 put_unaligned_le32(encryption_info->encrypt_tweak_upper,
5792 &request->encrypt_tweak_upper);
5793 }
5794
5795 rc = pqi_build_aio_sg_list(ctrl_info, request, scmd, io_request);
5796 if (rc) {
5797 pqi_free_io_request(io_request);
5798 return SCSI_MLQUEUE_HOST_BUSY;
5799 }
5800
5801 pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);
5802
5803 return 0;
5804}
5805
5806static int pqi_aio_submit_r1_write_io(struct pqi_ctrl_info *ctrl_info,
5807 struct scsi_cmnd *scmd, struct pqi_queue_group *queue_group,
5808 struct pqi_encryption_info *encryption_info, struct pqi_scsi_dev *device,
5809 struct pqi_scsi_dev_raid_map_data *rmd)
5810{
5811 int rc;
5812 struct pqi_io_request *io_request;
5813 struct pqi_aio_r1_path_request *r1_request;
5814
5815 io_request = pqi_alloc_io_request(ctrl_info, scmd);
5816 if (!io_request)
5817 return SCSI_MLQUEUE_HOST_BUSY;
5818
5819 io_request->io_complete_callback = pqi_aio_io_complete;
5820 io_request->scmd = scmd;
5821 io_request->raid_bypass = true;
5822
5823 r1_request = io_request->iu;
5824 memset(r1_request, 0, offsetof(struct pqi_aio_r1_path_request, sg_descriptors));
5825
5826 r1_request->header.iu_type = PQI_REQUEST_IU_AIO_PATH_RAID1_IO;
5827 put_unaligned_le16(*(u16 *)device->scsi3addr & 0x3fff, &r1_request->volume_id);
5828 r1_request->num_drives = rmd->num_it_nexus_entries;
5829 put_unaligned_le32(rmd->it_nexus[0], &r1_request->it_nexus_1);
5830 put_unaligned_le32(rmd->it_nexus[1], &r1_request->it_nexus_2);
5831 if (rmd->num_it_nexus_entries == 3)
5832 put_unaligned_le32(rmd->it_nexus[2], &r1_request->it_nexus_3);
5833
5834 put_unaligned_le32(scsi_bufflen(scmd), &r1_request->data_length);
5835 r1_request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
5836 put_unaligned_le16(io_request->index, &r1_request->request_id);
5837 r1_request->error_index = r1_request->request_id;
5838 if (rmd->cdb_length > sizeof(r1_request->cdb))
5839 rmd->cdb_length = sizeof(r1_request->cdb);
5840 r1_request->cdb_length = rmd->cdb_length;
5841 memcpy(r1_request->cdb, rmd->cdb, rmd->cdb_length);
5842
5843 /* The direction is always write. */
5844 r1_request->data_direction = SOP_READ_FLAG;
5845
5846 if (encryption_info) {
5847 r1_request->encryption_enable = true;
5848 put_unaligned_le16(encryption_info->data_encryption_key_index,
5849 &r1_request->data_encryption_key_index);
5850 put_unaligned_le32(encryption_info->encrypt_tweak_lower,
5851 &r1_request->encrypt_tweak_lower);
5852 put_unaligned_le32(encryption_info->encrypt_tweak_upper,
5853 &r1_request->encrypt_tweak_upper);
5854 }
5855
5856 rc = pqi_build_aio_r1_sg_list(ctrl_info, r1_request, scmd, io_request);
5857 if (rc) {
5858 pqi_free_io_request(io_request);
5859 return SCSI_MLQUEUE_HOST_BUSY;
5860 }
5861
5862 pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);
5863
5864 return 0;
5865}
5866
5867static int pqi_aio_submit_r56_write_io(struct pqi_ctrl_info *ctrl_info,
5868 struct scsi_cmnd *scmd, struct pqi_queue_group *queue_group,
5869 struct pqi_encryption_info *encryption_info, struct pqi_scsi_dev *device,
5870 struct pqi_scsi_dev_raid_map_data *rmd)
5871{
5872 int rc;
5873 struct pqi_io_request *io_request;
5874 struct pqi_aio_r56_path_request *r56_request;
5875
5876 io_request = pqi_alloc_io_request(ctrl_info, scmd);
5877 if (!io_request)
5878 return SCSI_MLQUEUE_HOST_BUSY;
5879 io_request->io_complete_callback = pqi_aio_io_complete;
5880 io_request->scmd = scmd;
5881 io_request->raid_bypass = true;
5882
5883 r56_request = io_request->iu;
5884 memset(r56_request, 0, offsetof(struct pqi_aio_r56_path_request, sg_descriptors));
5885
5886 if (device->raid_level == SA_RAID_5 || device->raid_level == SA_RAID_51)
5887 r56_request->header.iu_type = PQI_REQUEST_IU_AIO_PATH_RAID5_IO;
5888 else
5889 r56_request->header.iu_type = PQI_REQUEST_IU_AIO_PATH_RAID6_IO;
5890
5891 put_unaligned_le16(*(u16 *)device->scsi3addr & 0x3fff, &r56_request->volume_id);
5892 put_unaligned_le32(rmd->aio_handle, &r56_request->data_it_nexus);
5893 put_unaligned_le32(rmd->p_parity_it_nexus, &r56_request->p_parity_it_nexus);
5894 if (rmd->raid_level == SA_RAID_6) {
5895 put_unaligned_le32(rmd->q_parity_it_nexus, &r56_request->q_parity_it_nexus);
5896 r56_request->xor_multiplier = rmd->xor_mult;
5897 }
5898 put_unaligned_le32(scsi_bufflen(scmd), &r56_request->data_length);
5899 r56_request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
5900 put_unaligned_le64(rmd->row, &r56_request->row);
5901
5902 put_unaligned_le16(io_request->index, &r56_request->request_id);
5903 r56_request->error_index = r56_request->request_id;
5904
5905 if (rmd->cdb_length > sizeof(r56_request->cdb))
5906 rmd->cdb_length = sizeof(r56_request->cdb);
5907 r56_request->cdb_length = rmd->cdb_length;
5908 memcpy(r56_request->cdb, rmd->cdb, rmd->cdb_length);
5909
5910 /* The direction is always write. */
5911 r56_request->data_direction = SOP_READ_FLAG;
5912
5913 if (encryption_info) {
5914 r56_request->encryption_enable = true;
5915 put_unaligned_le16(encryption_info->data_encryption_key_index,
5916 &r56_request->data_encryption_key_index);
5917 put_unaligned_le32(encryption_info->encrypt_tweak_lower,
5918 &r56_request->encrypt_tweak_lower);
5919 put_unaligned_le32(encryption_info->encrypt_tweak_upper,
5920 &r56_request->encrypt_tweak_upper);
5921 }
5922
5923 rc = pqi_build_aio_r56_sg_list(ctrl_info, r56_request, scmd, io_request);
5924 if (rc) {
5925 pqi_free_io_request(io_request);
5926 return SCSI_MLQUEUE_HOST_BUSY;
5927 }
5928
5929 pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);
5930
5931 return 0;
5932}
5933
5934static inline u16 pqi_get_hw_queue(struct pqi_ctrl_info *ctrl_info,
5935 struct scsi_cmnd *scmd)
5936{
5937 /*
5938 * We are setting host_tagset = 1 during init.
5939 */
5940 return blk_mq_unique_tag_to_hwq(blk_mq_unique_tag(scsi_cmd_to_rq(scmd)));
5941}
5942
5943static inline bool pqi_is_bypass_eligible_request(struct scsi_cmnd *scmd)
5944{
5945 if (blk_rq_is_passthrough(scsi_cmd_to_rq(scmd)))
5946 return false;
5947
5948 return pqi_cmd_priv(scmd)->this_residual == 0;
5949}
5950
5951/*
5952 * This function gets called just before we hand the completed SCSI request
5953 * back to the SML.
5954 */
5955
5956void pqi_prep_for_scsi_done(struct scsi_cmnd *scmd)
5957{
5958 struct pqi_scsi_dev *device;
5959 struct completion *wait;
5960
5961 if (!scmd->device) {
5962 set_host_byte(scmd, DID_NO_CONNECT);
5963 return;
5964 }
5965
5966 device = scmd->device->hostdata;
5967 if (!device) {
5968 set_host_byte(scmd, DID_NO_CONNECT);
5969 return;
5970 }
5971
5972 atomic_dec(&device->scsi_cmds_outstanding[scmd->device->lun]);
5973
5974 wait = (struct completion *)xchg(&scmd->host_scribble, NULL);
5975 if (wait != PQI_NO_COMPLETION)
5976 complete(wait);
5977}
5978
5979static bool pqi_is_parity_write_stream(struct pqi_ctrl_info *ctrl_info,
5980 struct scsi_cmnd *scmd)
5981{
5982 u32 oldest_jiffies;
5983 u8 lru_index;
5984 int i;
5985 int rc;
5986 struct pqi_scsi_dev *device;
5987 struct pqi_stream_data *pqi_stream_data;
5988 struct pqi_scsi_dev_raid_map_data rmd = { 0 };
5989
5990 if (!ctrl_info->enable_stream_detection)
5991 return false;
5992
5993 rc = pqi_get_aio_lba_and_block_count(scmd, &rmd);
5994 if (rc)
5995 return false;
5996
5997 /* Check writes only. */
5998 if (!rmd.is_write)
5999 return false;
6000
6001 device = scmd->device->hostdata;
6002
6003 /* Check for RAID 5/6 streams. */
6004 if (device->raid_level != SA_RAID_5 && device->raid_level != SA_RAID_6)
6005 return false;
6006
6007 /*
6008 * If controller does not support AIO RAID{5,6} writes, need to send
6009 * requests down non-AIO path.
6010 */
6011 if ((device->raid_level == SA_RAID_5 && !ctrl_info->enable_r5_writes) ||
6012 (device->raid_level == SA_RAID_6 && !ctrl_info->enable_r6_writes))
6013 return true;
6014
6015 lru_index = 0;
6016 oldest_jiffies = INT_MAX;
6017 for (i = 0; i < NUM_STREAMS_PER_LUN; i++) {
6018 pqi_stream_data = &device->stream_data[i];
6019 /*
6020 * Check for adjacent request or request is within
6021 * the previous request.
6022 */
6023 if ((pqi_stream_data->next_lba &&
6024 rmd.first_block >= pqi_stream_data->next_lba) &&
6025 rmd.first_block <= pqi_stream_data->next_lba +
6026 rmd.block_cnt) {
6027 pqi_stream_data->next_lba = rmd.first_block +
6028 rmd.block_cnt;
6029 pqi_stream_data->last_accessed = jiffies;
6030 per_cpu_ptr(device->raid_io_stats, raw_smp_processor_id())->write_stream_cnt++;
6031 return true;
6032 }
6033
6034 /* unused entry */
6035 if (pqi_stream_data->last_accessed == 0) {
6036 lru_index = i;
6037 break;
6038 }
6039
6040 /* Find entry with oldest last accessed time. */
6041 if (pqi_stream_data->last_accessed <= oldest_jiffies) {
6042 oldest_jiffies = pqi_stream_data->last_accessed;
6043 lru_index = i;
6044 }
6045 }
6046
6047 /* Set LRU entry. */
6048 pqi_stream_data = &device->stream_data[lru_index];
6049 pqi_stream_data->last_accessed = jiffies;
6050 pqi_stream_data->next_lba = rmd.first_block + rmd.block_cnt;
6051
6052 return false;
6053}
6054
6055static enum scsi_qc_status pqi_scsi_queue_command(struct Scsi_Host *shost,
6056 struct scsi_cmnd *scmd)
6057{
6058 int rc;
6059 struct pqi_ctrl_info *ctrl_info;
6060 struct pqi_scsi_dev *device;
6061 u16 hw_queue;
6062 struct pqi_queue_group *queue_group;
6063 bool raid_bypassed;
6064 u8 lun;
6065
6066 scmd->host_scribble = PQI_NO_COMPLETION;
6067
6068 device = scmd->device->hostdata;
6069
6070 if (!device) {
6071 set_host_byte(scmd, DID_NO_CONNECT);
6072 pqi_scsi_done(scmd);
6073 return 0;
6074 }
6075
6076 lun = (u8)scmd->device->lun;
6077
6078 atomic_inc(&device->scsi_cmds_outstanding[lun]);
6079
6080 ctrl_info = shost_to_hba(shost);
6081
6082 if (pqi_ctrl_offline(ctrl_info) || pqi_device_offline(device) || pqi_device_in_remove(device)) {
6083 set_host_byte(scmd, DID_NO_CONNECT);
6084 pqi_scsi_done(scmd);
6085 return 0;
6086 }
6087
6088 if (pqi_ctrl_blocked(ctrl_info) || pqi_device_in_reset(device, lun)) {
6089 rc = SCSI_MLQUEUE_HOST_BUSY;
6090 goto out;
6091 }
6092
6093 /*
6094 * This is necessary because the SML doesn't zero out this field during
6095 * error recovery.
6096 */
6097 scmd->result = 0;
6098
6099 hw_queue = pqi_get_hw_queue(ctrl_info, scmd);
6100 queue_group = &ctrl_info->queue_groups[hw_queue];
6101
6102 if (pqi_is_logical_device(device)) {
6103 raid_bypassed = false;
6104 if (device->raid_bypass_enabled &&
6105 pqi_is_bypass_eligible_request(scmd) &&
6106 !pqi_is_parity_write_stream(ctrl_info, scmd)) {
6107 rc = pqi_raid_bypass_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);
6108 if (rc == 0 || rc == SCSI_MLQUEUE_HOST_BUSY) {
6109 raid_bypassed = true;
6110 per_cpu_ptr(device->raid_io_stats, raw_smp_processor_id())->raid_bypass_cnt++;
6111 }
6112 }
6113 if (!raid_bypassed)
6114 rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);
6115 } else {
6116 if (device->aio_enabled)
6117 rc = pqi_aio_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);
6118 else
6119 rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);
6120 }
6121
6122out:
6123 if (rc) {
6124 scmd->host_scribble = NULL;
6125 atomic_dec(&device->scsi_cmds_outstanding[lun]);
6126 }
6127
6128 return rc;
6129}
6130
6131static unsigned int pqi_queued_io_count(struct pqi_ctrl_info *ctrl_info)
6132{
6133 unsigned int i;
6134 unsigned int path;
6135 unsigned long flags;
6136 unsigned int queued_io_count;
6137 struct pqi_queue_group *queue_group;
6138 struct pqi_io_request *io_request;
6139
6140 queued_io_count = 0;
6141
6142 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
6143 queue_group = &ctrl_info->queue_groups[i];
6144 for (path = 0; path < 2; path++) {
6145 spin_lock_irqsave(&queue_group->submit_lock[path], flags);
6146 list_for_each_entry(io_request, &queue_group->request_list[path], request_list_entry)
6147 queued_io_count++;
6148 spin_unlock_irqrestore(&queue_group->submit_lock[path], flags);
6149 }
6150 }
6151
6152 return queued_io_count;
6153}
6154
6155static unsigned int pqi_nonempty_inbound_queue_count(struct pqi_ctrl_info *ctrl_info)
6156{
6157 unsigned int i;
6158 unsigned int path;
6159 unsigned int nonempty_inbound_queue_count;
6160 struct pqi_queue_group *queue_group;
6161 pqi_index_t iq_pi;
6162 pqi_index_t iq_ci;
6163
6164 nonempty_inbound_queue_count = 0;
6165
6166 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
6167 queue_group = &ctrl_info->queue_groups[i];
6168 for (path = 0; path < 2; path++) {
6169 iq_pi = queue_group->iq_pi_copy[path];
6170 iq_ci = readl(queue_group->iq_ci[path]);
6171 if (iq_ci != iq_pi)
6172 nonempty_inbound_queue_count++;
6173 }
6174 }
6175
6176 return nonempty_inbound_queue_count;
6177}
6178
6179#define PQI_INBOUND_QUEUES_NONEMPTY_WARNING_TIMEOUT_SECS 10
6180
6181static int pqi_wait_until_inbound_queues_empty(struct pqi_ctrl_info *ctrl_info)
6182{
6183 unsigned long start_jiffies;
6184 unsigned long warning_timeout;
6185 unsigned int queued_io_count;
6186 unsigned int nonempty_inbound_queue_count;
6187 bool displayed_warning;
6188
6189 displayed_warning = false;
6190 start_jiffies = jiffies;
6191 warning_timeout = (PQI_INBOUND_QUEUES_NONEMPTY_WARNING_TIMEOUT_SECS * HZ) + start_jiffies;
6192
6193 while (1) {
6194 queued_io_count = pqi_queued_io_count(ctrl_info);
6195 nonempty_inbound_queue_count = pqi_nonempty_inbound_queue_count(ctrl_info);
6196 if (queued_io_count == 0 && nonempty_inbound_queue_count == 0)
6197 break;
6198 pqi_check_ctrl_health(ctrl_info);
6199 if (pqi_ctrl_offline(ctrl_info))
6200 return -ENXIO;
6201 if (time_after(jiffies, warning_timeout)) {
6202 dev_warn(&ctrl_info->pci_dev->dev,
6203 "waiting %u seconds for queued I/O to drain (queued I/O count: %u; non-empty inbound queue count: %u)\n",
6204 jiffies_to_msecs(jiffies - start_jiffies) / 1000, queued_io_count, nonempty_inbound_queue_count);
6205 displayed_warning = true;
6206 warning_timeout = (PQI_INBOUND_QUEUES_NONEMPTY_WARNING_TIMEOUT_SECS * HZ) + jiffies;
6207 }
6208 usleep_range(1000, 2000);
6209 }
6210
6211 if (displayed_warning)
6212 dev_warn(&ctrl_info->pci_dev->dev,
6213 "queued I/O drained after waiting for %u seconds\n",
6214 jiffies_to_msecs(jiffies - start_jiffies) / 1000);
6215
6216 return 0;
6217}
6218
6219static void pqi_fail_io_queued_for_device(struct pqi_ctrl_info *ctrl_info,
6220 struct pqi_scsi_dev *device, u8 lun)
6221{
6222 unsigned int i;
6223 unsigned int path;
6224 struct pqi_queue_group *queue_group;
6225 unsigned long flags;
6226 struct pqi_io_request *io_request;
6227 struct pqi_io_request *next;
6228 struct scsi_cmnd *scmd;
6229 struct pqi_scsi_dev *scsi_device;
6230
6231 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
6232 queue_group = &ctrl_info->queue_groups[i];
6233
6234 for (path = 0; path < 2; path++) {
6235 spin_lock_irqsave(
6236 &queue_group->submit_lock[path], flags);
6237
6238 list_for_each_entry_safe(io_request, next,
6239 &queue_group->request_list[path],
6240 request_list_entry) {
6241
6242 scmd = io_request->scmd;
6243 if (!scmd)
6244 continue;
6245
6246 scsi_device = scmd->device->hostdata;
6247
6248 list_del(&io_request->request_list_entry);
6249 if (scsi_device == device && (u8)scmd->device->lun == lun)
6250 set_host_byte(scmd, DID_RESET);
6251 else
6252 set_host_byte(scmd, DID_REQUEUE);
6253 pqi_free_io_request(io_request);
6254 scsi_dma_unmap(scmd);
6255 pqi_scsi_done(scmd);
6256 }
6257
6258 spin_unlock_irqrestore(
6259 &queue_group->submit_lock[path], flags);
6260 }
6261 }
6262}
6263
6264#define PQI_PENDING_IO_WARNING_TIMEOUT_SECS 10
6265
6266static int pqi_device_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info,
6267 struct pqi_scsi_dev *device, u8 lun, unsigned long timeout_msecs)
6268{
6269 int cmds_outstanding;
6270 unsigned long start_jiffies;
6271 unsigned long warning_timeout;
6272 unsigned long msecs_waiting;
6273
6274 start_jiffies = jiffies;
6275 warning_timeout = (PQI_PENDING_IO_WARNING_TIMEOUT_SECS * HZ) + start_jiffies;
6276
6277 while ((cmds_outstanding = atomic_read(&device->scsi_cmds_outstanding[lun])) > 0) {
6278 if (ctrl_info->ctrl_removal_state != PQI_CTRL_GRACEFUL_REMOVAL) {
6279 pqi_check_ctrl_health(ctrl_info);
6280 if (pqi_ctrl_offline(ctrl_info))
6281 return -ENXIO;
6282 }
6283 msecs_waiting = jiffies_to_msecs(jiffies - start_jiffies);
6284 if (msecs_waiting >= timeout_msecs) {
6285 dev_err(&ctrl_info->pci_dev->dev,
6286 "scsi %d:%d:%d:%d: timed out after %lu seconds waiting for %d outstanding command(s)\n",
6287 ctrl_info->scsi_host->host_no, device->bus, device->target,
6288 lun, msecs_waiting / 1000, cmds_outstanding);
6289 return -ETIMEDOUT;
6290 }
6291 if (time_after(jiffies, warning_timeout)) {
6292 dev_warn(&ctrl_info->pci_dev->dev,
6293 "scsi %d:%d:%d:%d: waiting %lu seconds for %d outstanding command(s)\n",
6294 ctrl_info->scsi_host->host_no, device->bus, device->target,
6295 lun, msecs_waiting / 1000, cmds_outstanding);
6296 warning_timeout = (PQI_PENDING_IO_WARNING_TIMEOUT_SECS * HZ) + jiffies;
6297 }
6298 usleep_range(1000, 2000);
6299 }
6300
6301 return 0;
6302}
6303
6304static void pqi_lun_reset_complete(struct pqi_io_request *io_request,
6305 void *context)
6306{
6307 struct completion *waiting = context;
6308
6309 complete(waiting);
6310}
6311
6312#define PQI_LUN_RESET_POLL_COMPLETION_SECS 10
6313
6314static int pqi_wait_for_lun_reset_completion(struct pqi_ctrl_info *ctrl_info,
6315 struct pqi_scsi_dev *device, u8 lun, struct completion *wait)
6316{
6317 int rc;
6318 unsigned int wait_secs;
6319 int cmds_outstanding;
6320
6321 wait_secs = 0;
6322
6323 while (1) {
6324 if (wait_for_completion_io_timeout(wait,
6325 PQI_LUN_RESET_POLL_COMPLETION_SECS * HZ)) {
6326 rc = 0;
6327 break;
6328 }
6329
6330 pqi_check_ctrl_health(ctrl_info);
6331 if (pqi_ctrl_offline(ctrl_info)) {
6332 rc = -ENXIO;
6333 break;
6334 }
6335
6336 wait_secs += PQI_LUN_RESET_POLL_COMPLETION_SECS;
6337 cmds_outstanding = atomic_read(&device->scsi_cmds_outstanding[lun]);
6338 dev_warn(&ctrl_info->pci_dev->dev,
6339 "scsi %d:%d:%d:%d: waiting %u seconds for LUN reset to complete (%d command(s) outstanding)\n",
6340 ctrl_info->scsi_host->host_no, device->bus, device->target, lun, wait_secs, cmds_outstanding);
6341 }
6342
6343 return rc;
6344}
6345
6346#define PQI_LUN_RESET_FIRMWARE_TIMEOUT_SECS 30
6347
6348static int pqi_lun_reset(struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev *device, u8 lun)
6349{
6350 int rc;
6351 struct pqi_io_request *io_request;
6352 DECLARE_COMPLETION_ONSTACK(wait);
6353 struct pqi_task_management_request *request;
6354
6355 io_request = pqi_alloc_io_request(ctrl_info, NULL);
6356 io_request->io_complete_callback = pqi_lun_reset_complete;
6357 io_request->context = &wait;
6358
6359 request = io_request->iu;
6360 memset(request, 0, sizeof(*request));
6361
6362 request->header.iu_type = PQI_REQUEST_IU_TASK_MANAGEMENT;
6363 put_unaligned_le16(sizeof(*request) - PQI_REQUEST_HEADER_LENGTH,
6364 &request->header.iu_length);
6365 put_unaligned_le16(io_request->index, &request->request_id);
6366 memcpy(request->lun_number, device->scsi3addr,
6367 sizeof(request->lun_number));
6368 if (!pqi_is_logical_device(device) && ctrl_info->multi_lun_device_supported)
6369 request->ml_device_lun_number = lun;
6370 request->task_management_function = SOP_TASK_MANAGEMENT_LUN_RESET;
6371 if (ctrl_info->tmf_iu_timeout_supported)
6372 put_unaligned_le16(PQI_LUN_RESET_FIRMWARE_TIMEOUT_SECS, &request->timeout);
6373
6374 pqi_start_io(ctrl_info, &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,
6375 io_request);
6376
6377 rc = pqi_wait_for_lun_reset_completion(ctrl_info, device, lun, &wait);
6378 if (rc == 0)
6379 rc = io_request->status;
6380
6381 pqi_free_io_request(io_request);
6382
6383 return rc;
6384}
6385
6386#define PQI_LUN_RESET_RETRIES 3
6387#define PQI_LUN_RESET_RETRY_INTERVAL_MSECS (10 * 1000)
6388#define PQI_LUN_RESET_PENDING_IO_TIMEOUT_MSECS (10 * 60 * 1000)
6389#define PQI_LUN_RESET_FAILED_PENDING_IO_TIMEOUT_MSECS (2 * 60 * 1000)
6390
6391static int pqi_lun_reset_with_retries(struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev *device, u8 lun)
6392{
6393 int reset_rc;
6394 int wait_rc;
6395 unsigned int retries;
6396 unsigned long timeout_msecs;
6397
6398 for (retries = 0;;) {
6399 reset_rc = pqi_lun_reset(ctrl_info, device, lun);
6400 if (reset_rc == 0 || reset_rc == -ENODEV || reset_rc == -ENXIO || ++retries > PQI_LUN_RESET_RETRIES)
6401 break;
6402 msleep(PQI_LUN_RESET_RETRY_INTERVAL_MSECS);
6403 }
6404
6405 timeout_msecs = reset_rc ? PQI_LUN_RESET_FAILED_PENDING_IO_TIMEOUT_MSECS :
6406 PQI_LUN_RESET_PENDING_IO_TIMEOUT_MSECS;
6407
6408 wait_rc = pqi_device_wait_for_pending_io(ctrl_info, device, lun, timeout_msecs);
6409 if (wait_rc && reset_rc == 0)
6410 reset_rc = wait_rc;
6411
6412 return reset_rc == 0 ? SUCCESS : FAILED;
6413}
6414
6415static int pqi_device_reset(struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev *device, u8 lun)
6416{
6417 int rc;
6418
6419 pqi_ctrl_block_requests(ctrl_info);
6420 pqi_ctrl_wait_until_quiesced(ctrl_info);
6421 pqi_fail_io_queued_for_device(ctrl_info, device, lun);
6422 rc = pqi_wait_until_inbound_queues_empty(ctrl_info);
6423 pqi_device_reset_start(device, lun);
6424 pqi_ctrl_unblock_requests(ctrl_info);
6425 if (rc)
6426 rc = FAILED;
6427 else
6428 rc = pqi_lun_reset_with_retries(ctrl_info, device, lun);
6429 pqi_device_reset_done(device, lun);
6430
6431 return rc;
6432}
6433
6434static int pqi_device_reset_handler(struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev *device, u8 lun, struct scsi_cmnd *scmd, u8 scsi_opcode)
6435{
6436 unsigned long flags;
6437 int rc;
6438
6439 mutex_lock(&ctrl_info->lun_reset_mutex);
6440
6441 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6442 if (pqi_find_scsi_dev(ctrl_info, device->bus, device->target, device->lun) == NULL) {
6443 dev_warn(&ctrl_info->pci_dev->dev,
6444 "skipping reset of scsi %d:%d:%d:%u, device has been removed\n",
6445 ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun);
6446 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6447 mutex_unlock(&ctrl_info->lun_reset_mutex);
6448 return 0;
6449 }
6450 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6451
6452 dev_err(&ctrl_info->pci_dev->dev,
6453 "resetting scsi %d:%d:%d:%u SCSI cmd at %p due to cmd opcode 0x%02x\n",
6454 ctrl_info->scsi_host->host_no, device->bus, device->target, lun, scmd, scsi_opcode);
6455
6456 pqi_check_ctrl_health(ctrl_info);
6457 if (pqi_ctrl_offline(ctrl_info))
6458 rc = FAILED;
6459 else
6460 rc = pqi_device_reset(ctrl_info, device, lun);
6461
6462 dev_err(&ctrl_info->pci_dev->dev,
6463 "reset of scsi %d:%d:%d:%u: %s\n",
6464 ctrl_info->scsi_host->host_no, device->bus, device->target, lun,
6465 rc == SUCCESS ? "SUCCESS" : "FAILED");
6466
6467 mutex_unlock(&ctrl_info->lun_reset_mutex);
6468
6469 return rc;
6470}
6471
6472static int pqi_eh_device_reset_handler(struct scsi_cmnd *scmd)
6473{
6474 struct Scsi_Host *shost;
6475 struct pqi_ctrl_info *ctrl_info;
6476 struct pqi_scsi_dev *device;
6477 u8 scsi_opcode;
6478
6479 shost = scmd->device->host;
6480 ctrl_info = shost_to_hba(shost);
6481 device = scmd->device->hostdata;
6482 scsi_opcode = scmd->cmd_len > 0 ? scmd->cmnd[0] : 0xff;
6483
6484 return pqi_device_reset_handler(ctrl_info, device, (u8)scmd->device->lun, scmd, scsi_opcode);
6485}
6486
6487static void pqi_tmf_worker(struct work_struct *work)
6488{
6489 struct pqi_tmf_work *tmf_work;
6490 struct scsi_cmnd *scmd;
6491
6492 tmf_work = container_of(work, struct pqi_tmf_work, work_struct);
6493 scmd = (struct scsi_cmnd *)xchg(&tmf_work->scmd, NULL);
6494
6495 pqi_device_reset_handler(tmf_work->ctrl_info, tmf_work->device, tmf_work->lun, scmd, tmf_work->scsi_opcode);
6496}
6497
6498static int pqi_eh_abort_handler(struct scsi_cmnd *scmd)
6499{
6500 struct Scsi_Host *shost;
6501 struct pqi_ctrl_info *ctrl_info;
6502 struct pqi_scsi_dev *device;
6503 struct pqi_tmf_work *tmf_work;
6504 DECLARE_COMPLETION_ONSTACK(wait);
6505
6506 shost = scmd->device->host;
6507 ctrl_info = shost_to_hba(shost);
6508 device = scmd->device->hostdata;
6509
6510 dev_err(&ctrl_info->pci_dev->dev,
6511 "attempting TASK ABORT on scsi %d:%d:%d:%d for SCSI cmd at %p\n",
6512 shost->host_no, device->bus, device->target, (int)scmd->device->lun, scmd);
6513
6514 if (cmpxchg(&scmd->host_scribble, PQI_NO_COMPLETION, (void *)&wait) == NULL) {
6515 dev_err(&ctrl_info->pci_dev->dev,
6516 "scsi %d:%d:%d:%d for SCSI cmd at %p already completed\n",
6517 shost->host_no, device->bus, device->target, (int)scmd->device->lun, scmd);
6518 scmd->result = DID_RESET << 16;
6519 goto out;
6520 }
6521
6522 tmf_work = &device->tmf_work[scmd->device->lun];
6523
6524 if (cmpxchg(&tmf_work->scmd, NULL, scmd) == NULL) {
6525 tmf_work->ctrl_info = ctrl_info;
6526 tmf_work->device = device;
6527 tmf_work->lun = (u8)scmd->device->lun;
6528 tmf_work->scsi_opcode = scmd->cmd_len > 0 ? scmd->cmnd[0] : 0xff;
6529 schedule_work(&tmf_work->work_struct);
6530 }
6531
6532 wait_for_completion(&wait);
6533
6534 dev_err(&ctrl_info->pci_dev->dev,
6535 "TASK ABORT on scsi %d:%d:%d:%d for SCSI cmd at %p: SUCCESS\n",
6536 shost->host_no, device->bus, device->target, (int)scmd->device->lun, scmd);
6537
6538out:
6539
6540 return SUCCESS;
6541}
6542
6543static int pqi_sdev_init(struct scsi_device *sdev)
6544{
6545 struct pqi_scsi_dev *device;
6546 unsigned long flags;
6547 struct pqi_ctrl_info *ctrl_info;
6548 struct scsi_target *starget;
6549 struct sas_rphy *rphy;
6550
6551 ctrl_info = shost_to_hba(sdev->host);
6552
6553 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6554
6555 if (sdev_channel(sdev) == PQI_PHYSICAL_DEVICE_BUS) {
6556 starget = scsi_target(sdev);
6557 rphy = target_to_rphy(starget);
6558 device = pqi_find_device_by_sas_rphy(ctrl_info, rphy);
6559 if (device) {
6560 if (device->target_lun_valid) {
6561 device->ignore_device = true;
6562 } else {
6563 device->target = sdev_id(sdev);
6564 device->lun = sdev->lun;
6565 device->target_lun_valid = true;
6566 }
6567 }
6568 } else {
6569 device = pqi_find_scsi_dev(ctrl_info, sdev_channel(sdev),
6570 sdev_id(sdev), sdev->lun);
6571 }
6572
6573 if (device) {
6574 sdev->hostdata = device;
6575 device->sdev = sdev;
6576 if (device->queue_depth) {
6577 device->advertised_queue_depth = device->queue_depth;
6578 scsi_change_queue_depth(sdev,
6579 device->advertised_queue_depth);
6580 }
6581 if (pqi_is_logical_device(device)) {
6582 pqi_disable_write_same(sdev);
6583 } else {
6584 sdev->allow_restart = 1;
6585 if (device->device_type == SA_DEVICE_TYPE_NVME)
6586 pqi_disable_write_same(sdev);
6587 }
6588 }
6589
6590 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6591
6592 return 0;
6593}
6594
6595static void pqi_map_queues(struct Scsi_Host *shost)
6596{
6597 struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
6598
6599 if (!ctrl_info->disable_managed_interrupts)
6600 blk_mq_map_hw_queues(&shost->tag_set.map[HCTX_TYPE_DEFAULT],
6601 &ctrl_info->pci_dev->dev, 0);
6602 else
6603 blk_mq_map_queues(&shost->tag_set.map[HCTX_TYPE_DEFAULT]);
6604}
6605
6606static inline bool pqi_is_tape_changer_device(struct pqi_scsi_dev *device)
6607{
6608 return device->devtype == TYPE_TAPE || device->devtype == TYPE_MEDIUM_CHANGER;
6609}
6610
6611static int pqi_sdev_configure(struct scsi_device *sdev,
6612 struct queue_limits *lim)
6613{
6614 int rc = 0;
6615 struct pqi_scsi_dev *device;
6616
6617 device = sdev->hostdata;
6618 device->devtype = sdev->type;
6619
6620 if (pqi_is_tape_changer_device(device) && device->ignore_device) {
6621 rc = -ENXIO;
6622 device->ignore_device = false;
6623 }
6624
6625 return rc;
6626}
6627
6628static void pqi_sdev_destroy(struct scsi_device *sdev)
6629{
6630 struct pqi_ctrl_info *ctrl_info;
6631 struct pqi_scsi_dev *device;
6632 struct pqi_tmf_work *tmf_work;
6633 int mutex_acquired;
6634 unsigned int lun;
6635 unsigned long flags;
6636
6637 ctrl_info = shost_to_hba(sdev->host);
6638
6639 mutex_acquired = mutex_trylock(&ctrl_info->scan_mutex);
6640 if (!mutex_acquired)
6641 return;
6642
6643 device = sdev->hostdata;
6644 if (!device) {
6645 mutex_unlock(&ctrl_info->scan_mutex);
6646 return;
6647 }
6648
6649 device->lun_count--;
6650 if (device->lun_count > 0) {
6651 mutex_unlock(&ctrl_info->scan_mutex);
6652 return;
6653 }
6654
6655 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
6656 list_del(&device->scsi_device_list_entry);
6657 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
6658
6659 mutex_unlock(&ctrl_info->scan_mutex);
6660
6661 for (lun = 0, tmf_work = device->tmf_work; lun < PQI_MAX_LUNS_PER_DEVICE; lun++, tmf_work++)
6662 cancel_work_sync(&tmf_work->work_struct);
6663
6664 mutex_lock(&ctrl_info->lun_reset_mutex);
6665 pqi_dev_info(ctrl_info, "removed", device);
6666 pqi_free_device(device);
6667 mutex_unlock(&ctrl_info->lun_reset_mutex);
6668}
6669
6670static int pqi_getpciinfo_ioctl(struct pqi_ctrl_info *ctrl_info, void __user *arg)
6671{
6672 struct pci_dev *pci_dev;
6673 u32 subsystem_vendor;
6674 u32 subsystem_device;
6675 cciss_pci_info_struct pci_info;
6676
6677 if (!arg)
6678 return -EINVAL;
6679
6680 pci_dev = ctrl_info->pci_dev;
6681
6682 pci_info.domain = pci_domain_nr(pci_dev->bus);
6683 pci_info.bus = pci_dev->bus->number;
6684 pci_info.dev_fn = pci_dev->devfn;
6685 subsystem_vendor = pci_dev->subsystem_vendor;
6686 subsystem_device = pci_dev->subsystem_device;
6687 pci_info.board_id = ((subsystem_device << 16) & 0xffff0000) | subsystem_vendor;
6688
6689 if (copy_to_user(arg, &pci_info, sizeof(pci_info)))
6690 return -EFAULT;
6691
6692 return 0;
6693}
6694
6695static int pqi_getdrivver_ioctl(void __user *arg)
6696{
6697 u32 version;
6698
6699 if (!arg)
6700 return -EINVAL;
6701
6702 version = (DRIVER_MAJOR << 28) | (DRIVER_MINOR << 24) |
6703 (DRIVER_RELEASE << 16) | DRIVER_REVISION;
6704
6705 if (copy_to_user(arg, &version, sizeof(version)))
6706 return -EFAULT;
6707
6708 return 0;
6709}
6710
6711struct ciss_error_info {
6712 u8 scsi_status;
6713 int command_status;
6714 size_t sense_data_length;
6715};
6716
6717static void pqi_error_info_to_ciss(struct pqi_raid_error_info *pqi_error_info,
6718 struct ciss_error_info *ciss_error_info)
6719{
6720 int ciss_cmd_status;
6721 size_t sense_data_length;
6722
6723 switch (pqi_error_info->data_out_result) {
6724 case PQI_DATA_IN_OUT_GOOD:
6725 ciss_cmd_status = CISS_CMD_STATUS_SUCCESS;
6726 break;
6727 case PQI_DATA_IN_OUT_UNDERFLOW:
6728 ciss_cmd_status = CISS_CMD_STATUS_DATA_UNDERRUN;
6729 break;
6730 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW:
6731 ciss_cmd_status = CISS_CMD_STATUS_DATA_OVERRUN;
6732 break;
6733 case PQI_DATA_IN_OUT_PROTOCOL_ERROR:
6734 case PQI_DATA_IN_OUT_BUFFER_ERROR:
6735 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_DESCRIPTOR_AREA:
6736 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_BRIDGE:
6737 case PQI_DATA_IN_OUT_ERROR:
6738 ciss_cmd_status = CISS_CMD_STATUS_PROTOCOL_ERROR;
6739 break;
6740 case PQI_DATA_IN_OUT_HARDWARE_ERROR:
6741 case PQI_DATA_IN_OUT_PCIE_FABRIC_ERROR:
6742 case PQI_DATA_IN_OUT_PCIE_COMPLETION_TIMEOUT:
6743 case PQI_DATA_IN_OUT_PCIE_COMPLETER_ABORT_RECEIVED:
6744 case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST_RECEIVED:
6745 case PQI_DATA_IN_OUT_PCIE_ECRC_CHECK_FAILED:
6746 case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST:
6747 case PQI_DATA_IN_OUT_PCIE_ACS_VIOLATION:
6748 case PQI_DATA_IN_OUT_PCIE_TLP_PREFIX_BLOCKED:
6749 case PQI_DATA_IN_OUT_PCIE_POISONED_MEMORY_READ:
6750 ciss_cmd_status = CISS_CMD_STATUS_HARDWARE_ERROR;
6751 break;
6752 case PQI_DATA_IN_OUT_UNSOLICITED_ABORT:
6753 ciss_cmd_status = CISS_CMD_STATUS_UNSOLICITED_ABORT;
6754 break;
6755 case PQI_DATA_IN_OUT_ABORTED:
6756 ciss_cmd_status = CISS_CMD_STATUS_ABORTED;
6757 break;
6758 case PQI_DATA_IN_OUT_TIMEOUT:
6759 ciss_cmd_status = CISS_CMD_STATUS_TIMEOUT;
6760 break;
6761 default:
6762 ciss_cmd_status = CISS_CMD_STATUS_TARGET_STATUS;
6763 break;
6764 }
6765
6766 sense_data_length =
6767 get_unaligned_le16(&pqi_error_info->sense_data_length);
6768 if (sense_data_length == 0)
6769 sense_data_length =
6770 get_unaligned_le16(&pqi_error_info->response_data_length);
6771 if (sense_data_length)
6772 if (sense_data_length > sizeof(pqi_error_info->data))
6773 sense_data_length = sizeof(pqi_error_info->data);
6774
6775 ciss_error_info->scsi_status = pqi_error_info->status;
6776 ciss_error_info->command_status = ciss_cmd_status;
6777 ciss_error_info->sense_data_length = sense_data_length;
6778}
6779
6780static int pqi_passthru_ioctl(struct pqi_ctrl_info *ctrl_info, void __user *arg)
6781{
6782 int rc;
6783 char *kernel_buffer = NULL;
6784 u16 iu_length;
6785 size_t sense_data_length;
6786 IOCTL_Command_struct iocommand;
6787 struct pqi_raid_path_request request;
6788 struct pqi_raid_error_info pqi_error_info;
6789 struct ciss_error_info ciss_error_info;
6790
6791 if (pqi_ctrl_offline(ctrl_info))
6792 return -ENXIO;
6793 if (pqi_ofa_in_progress(ctrl_info) && pqi_ctrl_blocked(ctrl_info))
6794 return -EBUSY;
6795 if (!arg)
6796 return -EINVAL;
6797 if (!capable(CAP_SYS_RAWIO))
6798 return -EPERM;
6799 if (copy_from_user(&iocommand, arg, sizeof(iocommand)))
6800 return -EFAULT;
6801 if (iocommand.buf_size < 1 &&
6802 iocommand.Request.Type.Direction != XFER_NONE)
6803 return -EINVAL;
6804 if (iocommand.Request.CDBLen > sizeof(request.cdb))
6805 return -EINVAL;
6806 if (iocommand.Request.Type.Type != TYPE_CMD)
6807 return -EINVAL;
6808
6809 switch (iocommand.Request.Type.Direction) {
6810 case XFER_NONE:
6811 case XFER_WRITE:
6812 case XFER_READ:
6813 case XFER_READ | XFER_WRITE:
6814 break;
6815 default:
6816 return -EINVAL;
6817 }
6818
6819 if (iocommand.buf_size > 0) {
6820 if (iocommand.Request.Type.Direction & XFER_WRITE) {
6821 kernel_buffer = memdup_user(iocommand.buf,
6822 iocommand.buf_size);
6823 if (IS_ERR(kernel_buffer))
6824 return PTR_ERR(kernel_buffer);
6825 } else {
6826 kernel_buffer = kzalloc(iocommand.buf_size, GFP_KERNEL);
6827 if (!kernel_buffer)
6828 return -ENOMEM;
6829 }
6830 }
6831
6832 memset(&request, 0, sizeof(request));
6833
6834 request.header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO;
6835 iu_length = offsetof(struct pqi_raid_path_request, sg_descriptors) -
6836 PQI_REQUEST_HEADER_LENGTH;
6837 memcpy(request.lun_number, iocommand.LUN_info.LunAddrBytes,
6838 sizeof(request.lun_number));
6839 memcpy(request.cdb, iocommand.Request.CDB, iocommand.Request.CDBLen);
6840 request.additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_0;
6841
6842 switch (iocommand.Request.Type.Direction) {
6843 case XFER_NONE:
6844 request.data_direction = SOP_NO_DIRECTION_FLAG;
6845 break;
6846 case XFER_WRITE:
6847 request.data_direction = SOP_WRITE_FLAG;
6848 break;
6849 case XFER_READ:
6850 request.data_direction = SOP_READ_FLAG;
6851 break;
6852 case XFER_READ | XFER_WRITE:
6853 request.data_direction = SOP_BIDIRECTIONAL;
6854 break;
6855 }
6856
6857 request.task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
6858
6859 if (iocommand.buf_size > 0) {
6860 put_unaligned_le32(iocommand.buf_size, &request.buffer_length);
6861
6862 rc = pqi_map_single(ctrl_info->pci_dev,
6863 &request.sg_descriptors[0], kernel_buffer,
6864 iocommand.buf_size, DMA_BIDIRECTIONAL);
6865 if (rc)
6866 goto out;
6867
6868 iu_length += sizeof(request.sg_descriptors[0]);
6869 }
6870
6871 put_unaligned_le16(iu_length, &request.header.iu_length);
6872
6873 if (ctrl_info->raid_iu_timeout_supported)
6874 put_unaligned_le32(iocommand.Request.Timeout, &request.timeout);
6875
6876 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,
6877 PQI_SYNC_FLAGS_INTERRUPTABLE, &pqi_error_info);
6878
6879 if (iocommand.buf_size > 0)
6880 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,
6881 DMA_BIDIRECTIONAL);
6882
6883 memset(&iocommand.error_info, 0, sizeof(iocommand.error_info));
6884
6885 if (rc == 0) {
6886 pqi_error_info_to_ciss(&pqi_error_info, &ciss_error_info);
6887 iocommand.error_info.ScsiStatus = ciss_error_info.scsi_status;
6888 iocommand.error_info.CommandStatus =
6889 ciss_error_info.command_status;
6890 sense_data_length = ciss_error_info.sense_data_length;
6891 if (sense_data_length) {
6892 if (sense_data_length >
6893 sizeof(iocommand.error_info.SenseInfo))
6894 sense_data_length =
6895 sizeof(iocommand.error_info.SenseInfo);
6896 memcpy(iocommand.error_info.SenseInfo,
6897 pqi_error_info.data, sense_data_length);
6898 iocommand.error_info.SenseLen = sense_data_length;
6899 }
6900 }
6901
6902 if (copy_to_user(arg, &iocommand, sizeof(iocommand))) {
6903 rc = -EFAULT;
6904 goto out;
6905 }
6906
6907 if (rc == 0 && iocommand.buf_size > 0 &&
6908 (iocommand.Request.Type.Direction & XFER_READ)) {
6909 if (copy_to_user(iocommand.buf, kernel_buffer,
6910 iocommand.buf_size)) {
6911 rc = -EFAULT;
6912 }
6913 }
6914
6915out:
6916 kfree(kernel_buffer);
6917
6918 return rc;
6919}
6920
6921static int pqi_ioctl(struct scsi_device *sdev, unsigned int cmd,
6922 void __user *arg)
6923{
6924 int rc;
6925 struct pqi_ctrl_info *ctrl_info;
6926
6927 ctrl_info = shost_to_hba(sdev->host);
6928
6929 switch (cmd) {
6930 case CCISS_DEREGDISK:
6931 case CCISS_REGNEWDISK:
6932 case CCISS_REGNEWD:
6933 rc = pqi_scan_scsi_devices(ctrl_info);
6934 break;
6935 case CCISS_GETPCIINFO:
6936 rc = pqi_getpciinfo_ioctl(ctrl_info, arg);
6937 break;
6938 case CCISS_GETDRIVVER:
6939 rc = pqi_getdrivver_ioctl(arg);
6940 break;
6941 case CCISS_PASSTHRU:
6942 rc = pqi_passthru_ioctl(ctrl_info, arg);
6943 break;
6944 default:
6945 rc = -EINVAL;
6946 break;
6947 }
6948
6949 return rc;
6950}
6951
6952static ssize_t pqi_firmware_version_show(struct device *dev,
6953 struct device_attribute *attr, char *buffer)
6954{
6955 struct Scsi_Host *shost;
6956 struct pqi_ctrl_info *ctrl_info;
6957
6958 shost = class_to_shost(dev);
6959 ctrl_info = shost_to_hba(shost);
6960
6961 return scnprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->firmware_version);
6962}
6963
6964static ssize_t pqi_serial_number_show(struct device *dev,
6965 struct device_attribute *attr, char *buffer)
6966{
6967 struct Scsi_Host *shost;
6968 struct pqi_ctrl_info *ctrl_info;
6969
6970 shost = class_to_shost(dev);
6971 ctrl_info = shost_to_hba(shost);
6972
6973 return scnprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->serial_number);
6974}
6975
6976static ssize_t pqi_model_show(struct device *dev,
6977 struct device_attribute *attr, char *buffer)
6978{
6979 struct Scsi_Host *shost;
6980 struct pqi_ctrl_info *ctrl_info;
6981
6982 shost = class_to_shost(dev);
6983 ctrl_info = shost_to_hba(shost);
6984
6985 return scnprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->model);
6986}
6987
6988static ssize_t pqi_vendor_show(struct device *dev,
6989 struct device_attribute *attr, char *buffer)
6990{
6991 struct Scsi_Host *shost;
6992 struct pqi_ctrl_info *ctrl_info;
6993
6994 shost = class_to_shost(dev);
6995 ctrl_info = shost_to_hba(shost);
6996
6997 return scnprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->vendor);
6998}
6999
7000static ssize_t pqi_host_rescan_store(struct device *dev,
7001 struct device_attribute *attr, const char *buffer, size_t count)
7002{
7003 struct Scsi_Host *shost = class_to_shost(dev);
7004
7005 pqi_scan_start(shost);
7006
7007 return count;
7008}
7009
7010static ssize_t pqi_lockup_action_show(struct device *dev,
7011 struct device_attribute *attr, char *buffer)
7012{
7013 int count = 0;
7014 unsigned int i;
7015
7016 for (i = 0; i < ARRAY_SIZE(pqi_lockup_actions); i++) {
7017 if (pqi_lockup_actions[i].action == pqi_lockup_action)
7018 count += scnprintf(buffer + count, PAGE_SIZE - count,
7019 "[%s] ", pqi_lockup_actions[i].name);
7020 else
7021 count += scnprintf(buffer + count, PAGE_SIZE - count,
7022 "%s ", pqi_lockup_actions[i].name);
7023 }
7024
7025 count += scnprintf(buffer + count, PAGE_SIZE - count, "\n");
7026
7027 return count;
7028}
7029
7030static ssize_t pqi_lockup_action_store(struct device *dev,
7031 struct device_attribute *attr, const char *buffer, size_t count)
7032{
7033 unsigned int i;
7034 char *action_name;
7035 char action_name_buffer[32];
7036
7037 strscpy(action_name_buffer, buffer, sizeof(action_name_buffer));
7038 action_name = strstrip(action_name_buffer);
7039
7040 for (i = 0; i < ARRAY_SIZE(pqi_lockup_actions); i++) {
7041 if (strcmp(action_name, pqi_lockup_actions[i].name) == 0) {
7042 pqi_lockup_action = pqi_lockup_actions[i].action;
7043 return count;
7044 }
7045 }
7046
7047 return -EINVAL;
7048}
7049
7050static ssize_t pqi_host_enable_stream_detection_show(struct device *dev,
7051 struct device_attribute *attr, char *buffer)
7052{
7053 struct Scsi_Host *shost = class_to_shost(dev);
7054 struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
7055
7056 return scnprintf(buffer, 10, "%x\n",
7057 ctrl_info->enable_stream_detection);
7058}
7059
7060static ssize_t pqi_host_enable_stream_detection_store(struct device *dev,
7061 struct device_attribute *attr, const char *buffer, size_t count)
7062{
7063 struct Scsi_Host *shost = class_to_shost(dev);
7064 struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
7065 u8 set_stream_detection = 0;
7066
7067 if (kstrtou8(buffer, 0, &set_stream_detection))
7068 return -EINVAL;
7069
7070 if (set_stream_detection > 0)
7071 set_stream_detection = 1;
7072
7073 ctrl_info->enable_stream_detection = set_stream_detection;
7074
7075 return count;
7076}
7077
7078static ssize_t pqi_host_enable_r5_writes_show(struct device *dev,
7079 struct device_attribute *attr, char *buffer)
7080{
7081 struct Scsi_Host *shost = class_to_shost(dev);
7082 struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
7083
7084 return scnprintf(buffer, 10, "%x\n", ctrl_info->enable_r5_writes);
7085}
7086
7087static ssize_t pqi_host_enable_r5_writes_store(struct device *dev,
7088 struct device_attribute *attr, const char *buffer, size_t count)
7089{
7090 struct Scsi_Host *shost = class_to_shost(dev);
7091 struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
7092 u8 set_r5_writes = 0;
7093
7094 if (kstrtou8(buffer, 0, &set_r5_writes))
7095 return -EINVAL;
7096
7097 if (set_r5_writes > 0)
7098 set_r5_writes = 1;
7099
7100 ctrl_info->enable_r5_writes = set_r5_writes;
7101
7102 return count;
7103}
7104
7105static ssize_t pqi_host_enable_r6_writes_show(struct device *dev,
7106 struct device_attribute *attr, char *buffer)
7107{
7108 struct Scsi_Host *shost = class_to_shost(dev);
7109 struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
7110
7111 return scnprintf(buffer, 10, "%x\n", ctrl_info->enable_r6_writes);
7112}
7113
7114static ssize_t pqi_host_enable_r6_writes_store(struct device *dev,
7115 struct device_attribute *attr, const char *buffer, size_t count)
7116{
7117 struct Scsi_Host *shost = class_to_shost(dev);
7118 struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
7119 u8 set_r6_writes = 0;
7120
7121 if (kstrtou8(buffer, 0, &set_r6_writes))
7122 return -EINVAL;
7123
7124 if (set_r6_writes > 0)
7125 set_r6_writes = 1;
7126
7127 ctrl_info->enable_r6_writes = set_r6_writes;
7128
7129 return count;
7130}
7131
7132static DEVICE_STRING_ATTR_RO(driver_version, 0444,
7133 DRIVER_VERSION BUILD_TIMESTAMP);
7134static DEVICE_ATTR(firmware_version, 0444, pqi_firmware_version_show, NULL);
7135static DEVICE_ATTR(model, 0444, pqi_model_show, NULL);
7136static DEVICE_ATTR(serial_number, 0444, pqi_serial_number_show, NULL);
7137static DEVICE_ATTR(vendor, 0444, pqi_vendor_show, NULL);
7138static DEVICE_ATTR(rescan, 0200, NULL, pqi_host_rescan_store);
7139static DEVICE_ATTR(lockup_action, 0644, pqi_lockup_action_show,
7140 pqi_lockup_action_store);
7141static DEVICE_ATTR(enable_stream_detection, 0644,
7142 pqi_host_enable_stream_detection_show,
7143 pqi_host_enable_stream_detection_store);
7144static DEVICE_ATTR(enable_r5_writes, 0644,
7145 pqi_host_enable_r5_writes_show, pqi_host_enable_r5_writes_store);
7146static DEVICE_ATTR(enable_r6_writes, 0644,
7147 pqi_host_enable_r6_writes_show, pqi_host_enable_r6_writes_store);
7148
7149static struct attribute *pqi_shost_attrs[] = {
7150 &dev_attr_driver_version.attr.attr,
7151 &dev_attr_firmware_version.attr,
7152 &dev_attr_model.attr,
7153 &dev_attr_serial_number.attr,
7154 &dev_attr_vendor.attr,
7155 &dev_attr_rescan.attr,
7156 &dev_attr_lockup_action.attr,
7157 &dev_attr_enable_stream_detection.attr,
7158 &dev_attr_enable_r5_writes.attr,
7159 &dev_attr_enable_r6_writes.attr,
7160 NULL
7161};
7162
7163ATTRIBUTE_GROUPS(pqi_shost);
7164
7165static ssize_t pqi_unique_id_show(struct device *dev,
7166 struct device_attribute *attr, char *buffer)
7167{
7168 struct pqi_ctrl_info *ctrl_info;
7169 struct scsi_device *sdev;
7170 struct pqi_scsi_dev *device;
7171 unsigned long flags;
7172 u8 unique_id[16];
7173
7174 sdev = to_scsi_device(dev);
7175 ctrl_info = shost_to_hba(sdev->host);
7176
7177 if (pqi_ctrl_offline(ctrl_info))
7178 return -ENODEV;
7179
7180 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
7181
7182 device = sdev->hostdata;
7183 if (!device) {
7184 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7185 return -ENODEV;
7186 }
7187
7188 if (device->is_physical_device)
7189 memcpy(unique_id, device->wwid, sizeof(device->wwid));
7190 else
7191 memcpy(unique_id, device->volume_id, sizeof(device->volume_id));
7192
7193 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7194
7195 return scnprintf(buffer, PAGE_SIZE,
7196 "%02X%02X%02X%02X%02X%02X%02X%02X"
7197 "%02X%02X%02X%02X%02X%02X%02X%02X\n",
7198 unique_id[0], unique_id[1], unique_id[2], unique_id[3],
7199 unique_id[4], unique_id[5], unique_id[6], unique_id[7],
7200 unique_id[8], unique_id[9], unique_id[10], unique_id[11],
7201 unique_id[12], unique_id[13], unique_id[14], unique_id[15]);
7202}
7203
7204static ssize_t pqi_lunid_show(struct device *dev,
7205 struct device_attribute *attr, char *buffer)
7206{
7207 struct pqi_ctrl_info *ctrl_info;
7208 struct scsi_device *sdev;
7209 struct pqi_scsi_dev *device;
7210 unsigned long flags;
7211 u8 lunid[8];
7212
7213 sdev = to_scsi_device(dev);
7214 ctrl_info = shost_to_hba(sdev->host);
7215
7216 if (pqi_ctrl_offline(ctrl_info))
7217 return -ENODEV;
7218
7219 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
7220
7221 device = sdev->hostdata;
7222 if (!device) {
7223 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7224 return -ENODEV;
7225 }
7226
7227 memcpy(lunid, device->scsi3addr, sizeof(lunid));
7228
7229 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7230
7231 return scnprintf(buffer, PAGE_SIZE, "0x%8phN\n", lunid);
7232}
7233
7234#define MAX_PATHS 8
7235
7236static ssize_t pqi_path_info_show(struct device *dev,
7237 struct device_attribute *attr, char *buf)
7238{
7239 struct pqi_ctrl_info *ctrl_info;
7240 struct scsi_device *sdev;
7241 struct pqi_scsi_dev *device;
7242 unsigned long flags;
7243 int i;
7244 int output_len = 0;
7245 u8 box;
7246 u8 bay;
7247 u8 path_map_index;
7248 char *active;
7249 u8 phys_connector[2];
7250
7251 sdev = to_scsi_device(dev);
7252 ctrl_info = shost_to_hba(sdev->host);
7253
7254 if (pqi_ctrl_offline(ctrl_info))
7255 return -ENODEV;
7256
7257 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
7258
7259 device = sdev->hostdata;
7260 if (!device) {
7261 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7262 return -ENODEV;
7263 }
7264
7265 bay = device->bay;
7266 for (i = 0; i < MAX_PATHS; i++) {
7267 path_map_index = 1 << i;
7268 if (i == device->active_path_index)
7269 active = "Active";
7270 else if (device->path_map & path_map_index)
7271 active = "Inactive";
7272 else
7273 continue;
7274
7275 output_len += scnprintf(buf + output_len,
7276 PAGE_SIZE - output_len,
7277 "[%d:%d:%d:%d] %20.20s ",
7278 ctrl_info->scsi_host->host_no,
7279 device->bus, device->target,
7280 device->lun,
7281 scsi_device_type(device->devtype));
7282
7283 if (device->devtype == TYPE_RAID ||
7284 pqi_is_logical_device(device))
7285 goto end_buffer;
7286
7287 memcpy(&phys_connector, &device->phys_connector[i],
7288 sizeof(phys_connector));
7289 if (phys_connector[0] < '0')
7290 phys_connector[0] = '0';
7291 if (phys_connector[1] < '0')
7292 phys_connector[1] = '0';
7293
7294 output_len += scnprintf(buf + output_len,
7295 PAGE_SIZE - output_len,
7296 "PORT: %.2s ", phys_connector);
7297
7298 box = device->box[i];
7299 if (box != 0 && box != 0xFF)
7300 output_len += scnprintf(buf + output_len,
7301 PAGE_SIZE - output_len,
7302 "BOX: %hhu ", box);
7303
7304 if ((device->devtype == TYPE_DISK ||
7305 device->devtype == TYPE_ZBC) &&
7306 pqi_expose_device(device))
7307 output_len += scnprintf(buf + output_len,
7308 PAGE_SIZE - output_len,
7309 "BAY: %hhu ", bay);
7310
7311end_buffer:
7312 output_len += scnprintf(buf + output_len,
7313 PAGE_SIZE - output_len,
7314 "%s\n", active);
7315 }
7316
7317 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7318
7319 return output_len;
7320}
7321
7322static ssize_t pqi_sas_address_show(struct device *dev,
7323 struct device_attribute *attr, char *buffer)
7324{
7325 struct pqi_ctrl_info *ctrl_info;
7326 struct scsi_device *sdev;
7327 struct pqi_scsi_dev *device;
7328 unsigned long flags;
7329 u64 sas_address;
7330
7331 sdev = to_scsi_device(dev);
7332 ctrl_info = shost_to_hba(sdev->host);
7333
7334 if (pqi_ctrl_offline(ctrl_info))
7335 return -ENODEV;
7336
7337 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
7338
7339 device = sdev->hostdata;
7340 if (!device) {
7341 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7342 return -ENODEV;
7343 }
7344
7345 sas_address = device->sas_address;
7346
7347 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7348
7349 return scnprintf(buffer, PAGE_SIZE, "0x%016llx\n", sas_address);
7350}
7351
7352static ssize_t pqi_ssd_smart_path_enabled_show(struct device *dev,
7353 struct device_attribute *attr, char *buffer)
7354{
7355 struct pqi_ctrl_info *ctrl_info;
7356 struct scsi_device *sdev;
7357 struct pqi_scsi_dev *device;
7358 unsigned long flags;
7359
7360 sdev = to_scsi_device(dev);
7361 ctrl_info = shost_to_hba(sdev->host);
7362
7363 if (pqi_ctrl_offline(ctrl_info))
7364 return -ENODEV;
7365
7366 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
7367
7368 device = sdev->hostdata;
7369 if (!device) {
7370 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7371 return -ENODEV;
7372 }
7373
7374 buffer[0] = device->raid_bypass_enabled ? '1' : '0';
7375 buffer[1] = '\n';
7376 buffer[2] = '\0';
7377
7378 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7379
7380 return 2;
7381}
7382
7383static ssize_t pqi_raid_level_show(struct device *dev,
7384 struct device_attribute *attr, char *buffer)
7385{
7386 struct pqi_ctrl_info *ctrl_info;
7387 struct scsi_device *sdev;
7388 struct pqi_scsi_dev *device;
7389 unsigned long flags;
7390 char *raid_level;
7391
7392 sdev = to_scsi_device(dev);
7393 ctrl_info = shost_to_hba(sdev->host);
7394
7395 if (pqi_ctrl_offline(ctrl_info))
7396 return -ENODEV;
7397
7398 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
7399
7400 device = sdev->hostdata;
7401 if (!device) {
7402 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7403 return -ENODEV;
7404 }
7405
7406 if (pqi_is_logical_device(device) && device->devtype == TYPE_DISK)
7407 raid_level = pqi_raid_level_to_string(device->raid_level);
7408 else
7409 raid_level = "N/A";
7410
7411 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7412
7413 return scnprintf(buffer, PAGE_SIZE, "%s\n", raid_level);
7414}
7415
7416static ssize_t pqi_raid_bypass_cnt_show(struct device *dev,
7417 struct device_attribute *attr, char *buffer)
7418{
7419 struct pqi_ctrl_info *ctrl_info;
7420 struct scsi_device *sdev;
7421 struct pqi_scsi_dev *device;
7422 unsigned long flags;
7423 u64 raid_bypass_cnt;
7424 int cpu;
7425
7426 sdev = to_scsi_device(dev);
7427 ctrl_info = shost_to_hba(sdev->host);
7428
7429 if (pqi_ctrl_offline(ctrl_info))
7430 return -ENODEV;
7431
7432 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
7433
7434 device = sdev->hostdata;
7435 if (!device) {
7436 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7437 return -ENODEV;
7438 }
7439
7440 raid_bypass_cnt = 0;
7441
7442 if (device->raid_io_stats) {
7443 for_each_online_cpu(cpu) {
7444 raid_bypass_cnt += per_cpu_ptr(device->raid_io_stats, cpu)->raid_bypass_cnt;
7445 }
7446 }
7447
7448 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7449
7450 return scnprintf(buffer, PAGE_SIZE, "0x%llx\n", raid_bypass_cnt);
7451}
7452
7453static ssize_t pqi_sas_ncq_prio_enable_show(struct device *dev,
7454 struct device_attribute *attr, char *buf)
7455{
7456 struct pqi_ctrl_info *ctrl_info;
7457 struct scsi_device *sdev;
7458 struct pqi_scsi_dev *device;
7459 unsigned long flags;
7460 int output_len = 0;
7461
7462 sdev = to_scsi_device(dev);
7463 ctrl_info = shost_to_hba(sdev->host);
7464
7465 if (pqi_ctrl_offline(ctrl_info))
7466 return -ENODEV;
7467
7468 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
7469
7470 device = sdev->hostdata;
7471 if (!device) {
7472 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7473 return -ENODEV;
7474 }
7475
7476 output_len = snprintf(buf, PAGE_SIZE, "%d\n",
7477 device->ncq_prio_enable);
7478 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7479
7480 return output_len;
7481}
7482
7483static ssize_t pqi_sas_ncq_prio_enable_store(struct device *dev,
7484 struct device_attribute *attr,
7485 const char *buf, size_t count)
7486{
7487 struct pqi_ctrl_info *ctrl_info;
7488 struct scsi_device *sdev;
7489 struct pqi_scsi_dev *device;
7490 unsigned long flags;
7491 u8 ncq_prio_enable = 0;
7492
7493 if (kstrtou8(buf, 0, &ncq_prio_enable))
7494 return -EINVAL;
7495
7496 sdev = to_scsi_device(dev);
7497 ctrl_info = shost_to_hba(sdev->host);
7498
7499 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
7500
7501 device = sdev->hostdata;
7502
7503 if (!device) {
7504 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7505 return -ENODEV;
7506 }
7507
7508 if (!device->ncq_prio_support) {
7509 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7510 return -EINVAL;
7511 }
7512
7513 device->ncq_prio_enable = ncq_prio_enable;
7514
7515 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7516
7517 return strlen(buf);
7518}
7519
7520static ssize_t pqi_numa_node_show(struct device *dev,
7521 struct device_attribute *attr, char *buffer)
7522{
7523 struct scsi_device *sdev;
7524 struct pqi_ctrl_info *ctrl_info;
7525
7526 sdev = to_scsi_device(dev);
7527 ctrl_info = shost_to_hba(sdev->host);
7528
7529 return scnprintf(buffer, PAGE_SIZE, "%d\n", ctrl_info->numa_node);
7530}
7531
7532static ssize_t pqi_write_stream_cnt_show(struct device *dev,
7533 struct device_attribute *attr, char *buffer)
7534{
7535 struct pqi_ctrl_info *ctrl_info;
7536 struct scsi_device *sdev;
7537 struct pqi_scsi_dev *device;
7538 unsigned long flags;
7539 u64 write_stream_cnt;
7540 int cpu;
7541
7542 sdev = to_scsi_device(dev);
7543 ctrl_info = shost_to_hba(sdev->host);
7544
7545 if (pqi_ctrl_offline(ctrl_info))
7546 return -ENODEV;
7547
7548 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
7549
7550 device = sdev->hostdata;
7551 if (!device) {
7552 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7553 return -ENODEV;
7554 }
7555
7556 write_stream_cnt = 0;
7557
7558 if (device->raid_io_stats) {
7559 for_each_online_cpu(cpu) {
7560 write_stream_cnt += per_cpu_ptr(device->raid_io_stats, cpu)->write_stream_cnt;
7561 }
7562 }
7563
7564 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
7565
7566 return scnprintf(buffer, PAGE_SIZE, "0x%llx\n", write_stream_cnt);
7567}
7568
7569static DEVICE_ATTR(lunid, 0444, pqi_lunid_show, NULL);
7570static DEVICE_ATTR(unique_id, 0444, pqi_unique_id_show, NULL);
7571static DEVICE_ATTR(path_info, 0444, pqi_path_info_show, NULL);
7572static DEVICE_ATTR(sas_address, 0444, pqi_sas_address_show, NULL);
7573static DEVICE_ATTR(ssd_smart_path_enabled, 0444, pqi_ssd_smart_path_enabled_show, NULL);
7574static DEVICE_ATTR(raid_level, 0444, pqi_raid_level_show, NULL);
7575static DEVICE_ATTR(raid_bypass_cnt, 0444, pqi_raid_bypass_cnt_show, NULL);
7576static DEVICE_ATTR(sas_ncq_prio_enable, 0644,
7577 pqi_sas_ncq_prio_enable_show, pqi_sas_ncq_prio_enable_store);
7578static DEVICE_ATTR(numa_node, 0444, pqi_numa_node_show, NULL);
7579static DEVICE_ATTR(write_stream_cnt, 0444, pqi_write_stream_cnt_show, NULL);
7580
7581static struct attribute *pqi_sdev_attrs[] = {
7582 &dev_attr_lunid.attr,
7583 &dev_attr_unique_id.attr,
7584 &dev_attr_path_info.attr,
7585 &dev_attr_sas_address.attr,
7586 &dev_attr_ssd_smart_path_enabled.attr,
7587 &dev_attr_raid_level.attr,
7588 &dev_attr_raid_bypass_cnt.attr,
7589 &dev_attr_sas_ncq_prio_enable.attr,
7590 &dev_attr_numa_node.attr,
7591 &dev_attr_write_stream_cnt.attr,
7592 NULL
7593};
7594
7595ATTRIBUTE_GROUPS(pqi_sdev);
7596
7597static const struct scsi_host_template pqi_driver_template = {
7598 .module = THIS_MODULE,
7599 .name = DRIVER_NAME_SHORT,
7600 .proc_name = DRIVER_NAME_SHORT,
7601 .queuecommand = pqi_scsi_queue_command,
7602 .scan_start = pqi_scan_start,
7603 .scan_finished = pqi_scan_finished,
7604 .this_id = -1,
7605 .eh_device_reset_handler = pqi_eh_device_reset_handler,
7606 .eh_abort_handler = pqi_eh_abort_handler,
7607 .ioctl = pqi_ioctl,
7608 .sdev_init = pqi_sdev_init,
7609 .sdev_configure = pqi_sdev_configure,
7610 .sdev_destroy = pqi_sdev_destroy,
7611 .map_queues = pqi_map_queues,
7612 .sdev_groups = pqi_sdev_groups,
7613 .shost_groups = pqi_shost_groups,
7614 .cmd_size = sizeof(struct pqi_cmd_priv),
7615};
7616
7617static int pqi_register_scsi(struct pqi_ctrl_info *ctrl_info)
7618{
7619 int rc;
7620 struct Scsi_Host *shost;
7621
7622 shost = scsi_host_alloc(&pqi_driver_template, sizeof(ctrl_info));
7623 if (!shost) {
7624 dev_err(&ctrl_info->pci_dev->dev, "scsi_host_alloc failed\n");
7625 return -ENOMEM;
7626 }
7627
7628 shost->io_port = 0;
7629 shost->n_io_port = 0;
7630 shost->this_id = -1;
7631 shost->max_channel = PQI_MAX_BUS;
7632 shost->max_cmd_len = MAX_COMMAND_SIZE;
7633 shost->max_lun = PQI_MAX_LUNS_PER_DEVICE;
7634 shost->max_id = ~0;
7635 shost->max_sectors = ctrl_info->max_sectors;
7636 shost->can_queue = ctrl_info->scsi_ml_can_queue;
7637 shost->cmd_per_lun = shost->can_queue;
7638 shost->sg_tablesize = ctrl_info->sg_tablesize;
7639 shost->transportt = pqi_sas_transport_template;
7640 shost->irq = pci_irq_vector(ctrl_info->pci_dev, 0);
7641 shost->unique_id = shost->irq;
7642 shost->nr_hw_queues = ctrl_info->num_queue_groups;
7643 shost->host_tagset = 1;
7644 shost->hostdata[0] = (unsigned long)ctrl_info;
7645
7646 rc = scsi_add_host(shost, &ctrl_info->pci_dev->dev);
7647 if (rc) {
7648 dev_err(&ctrl_info->pci_dev->dev, "scsi_add_host failed\n");
7649 goto free_host;
7650 }
7651
7652 rc = pqi_add_sas_host(shost, ctrl_info);
7653 if (rc) {
7654 dev_err(&ctrl_info->pci_dev->dev, "add SAS host failed\n");
7655 goto remove_host;
7656 }
7657
7658 ctrl_info->scsi_host = shost;
7659
7660 return 0;
7661
7662remove_host:
7663 scsi_remove_host(shost);
7664free_host:
7665 scsi_host_put(shost);
7666
7667 return rc;
7668}
7669
7670static void pqi_unregister_scsi(struct pqi_ctrl_info *ctrl_info)
7671{
7672 struct Scsi_Host *shost;
7673
7674 pqi_delete_sas_host(ctrl_info);
7675
7676 shost = ctrl_info->scsi_host;
7677 if (!shost)
7678 return;
7679
7680 scsi_remove_host(shost);
7681 scsi_host_put(shost);
7682}
7683
7684static int pqi_wait_for_pqi_reset_completion(struct pqi_ctrl_info *ctrl_info)
7685{
7686 int rc = 0;
7687 struct pqi_device_registers __iomem *pqi_registers;
7688 unsigned long timeout;
7689 unsigned int timeout_msecs;
7690 union pqi_reset_register reset_reg;
7691
7692 pqi_registers = ctrl_info->pqi_registers;
7693 timeout_msecs = readw(&pqi_registers->max_reset_timeout) * 100;
7694 timeout = msecs_to_jiffies(timeout_msecs) + jiffies;
7695
7696 while (1) {
7697 msleep(PQI_RESET_POLL_INTERVAL_MSECS);
7698 reset_reg.all_bits = readl(&pqi_registers->device_reset);
7699 if (reset_reg.bits.reset_action == PQI_RESET_ACTION_COMPLETED)
7700 break;
7701 if (!sis_is_firmware_running(ctrl_info)) {
7702 rc = -ENXIO;
7703 break;
7704 }
7705 if (time_after(jiffies, timeout)) {
7706 rc = -ETIMEDOUT;
7707 break;
7708 }
7709 }
7710
7711 return rc;
7712}
7713
7714static int pqi_reset(struct pqi_ctrl_info *ctrl_info)
7715{
7716 int rc;
7717 union pqi_reset_register reset_reg;
7718
7719 if (ctrl_info->pqi_reset_quiesce_supported) {
7720 rc = sis_pqi_reset_quiesce(ctrl_info);
7721 if (rc) {
7722 dev_err(&ctrl_info->pci_dev->dev,
7723 "PQI reset failed during quiesce with error %d\n", rc);
7724 return rc;
7725 }
7726 }
7727
7728 reset_reg.all_bits = 0;
7729 reset_reg.bits.reset_type = PQI_RESET_TYPE_HARD_RESET;
7730 reset_reg.bits.reset_action = PQI_RESET_ACTION_RESET;
7731
7732 writel(reset_reg.all_bits, &ctrl_info->pqi_registers->device_reset);
7733
7734 rc = pqi_wait_for_pqi_reset_completion(ctrl_info);
7735 if (rc)
7736 dev_err(&ctrl_info->pci_dev->dev,
7737 "PQI reset failed with error %d\n", rc);
7738
7739 return rc;
7740}
7741
7742static int pqi_get_ctrl_serial_number(struct pqi_ctrl_info *ctrl_info)
7743{
7744 int rc;
7745 struct bmic_sense_subsystem_info *sense_info;
7746
7747 sense_info = kzalloc_obj(*sense_info);
7748 if (!sense_info)
7749 return -ENOMEM;
7750
7751 rc = pqi_sense_subsystem_info(ctrl_info, sense_info);
7752 if (rc)
7753 goto out;
7754
7755 memcpy(ctrl_info->serial_number, sense_info->ctrl_serial_number,
7756 sizeof(sense_info->ctrl_serial_number));
7757 ctrl_info->serial_number[sizeof(sense_info->ctrl_serial_number)] = '\0';
7758
7759out:
7760 kfree(sense_info);
7761
7762 return rc;
7763}
7764
7765static int pqi_get_ctrl_product_details(struct pqi_ctrl_info *ctrl_info)
7766{
7767 int rc;
7768 struct bmic_identify_controller *identify;
7769
7770 identify = kmalloc_obj(*identify);
7771 if (!identify)
7772 return -ENOMEM;
7773
7774 rc = pqi_identify_controller(ctrl_info, identify);
7775 if (rc)
7776 goto out;
7777
7778 if (get_unaligned_le32(&identify->extra_controller_flags) &
7779 BMIC_IDENTIFY_EXTRA_FLAGS_LONG_FW_VERSION_SUPPORTED) {
7780 memcpy(ctrl_info->firmware_version,
7781 identify->firmware_version_long,
7782 sizeof(identify->firmware_version_long));
7783 } else {
7784 memcpy(ctrl_info->firmware_version,
7785 identify->firmware_version_short,
7786 sizeof(identify->firmware_version_short));
7787 ctrl_info->firmware_version
7788 [sizeof(identify->firmware_version_short)] = '\0';
7789 snprintf(ctrl_info->firmware_version +
7790 strlen(ctrl_info->firmware_version),
7791 sizeof(ctrl_info->firmware_version) -
7792 sizeof(identify->firmware_version_short),
7793 "-%u",
7794 get_unaligned_le16(&identify->firmware_build_number));
7795 }
7796
7797 memcpy(ctrl_info->model, identify->product_id,
7798 sizeof(identify->product_id));
7799 ctrl_info->model[sizeof(identify->product_id)] = '\0';
7800
7801 memcpy(ctrl_info->vendor, identify->vendor_id,
7802 sizeof(identify->vendor_id));
7803 ctrl_info->vendor[sizeof(identify->vendor_id)] = '\0';
7804
7805 dev_info(&ctrl_info->pci_dev->dev,
7806 "Firmware version: %s\n", ctrl_info->firmware_version);
7807
7808out:
7809 kfree(identify);
7810
7811 return rc;
7812}
7813
7814struct pqi_config_table_section_info {
7815 struct pqi_ctrl_info *ctrl_info;
7816 void *section;
7817 u32 section_offset;
7818 void __iomem *section_iomem_addr;
7819};
7820
7821static inline bool pqi_is_firmware_feature_supported(
7822 struct pqi_config_table_firmware_features *firmware_features,
7823 unsigned int bit_position)
7824{
7825 unsigned int byte_index;
7826
7827 byte_index = bit_position / BITS_PER_BYTE;
7828
7829 if (byte_index >= le16_to_cpu(firmware_features->num_elements))
7830 return false;
7831
7832 return firmware_features->features_supported[byte_index] &
7833 (1 << (bit_position % BITS_PER_BYTE)) ? true : false;
7834}
7835
7836static inline bool pqi_is_firmware_feature_enabled(
7837 struct pqi_config_table_firmware_features *firmware_features,
7838 void __iomem *firmware_features_iomem_addr,
7839 unsigned int bit_position)
7840{
7841 unsigned int byte_index;
7842 u8 __iomem *features_enabled_iomem_addr;
7843
7844 byte_index = (bit_position / BITS_PER_BYTE) +
7845 (le16_to_cpu(firmware_features->num_elements) * 2);
7846
7847 features_enabled_iomem_addr = firmware_features_iomem_addr +
7848 offsetof(struct pqi_config_table_firmware_features,
7849 features_supported) + byte_index;
7850
7851 return *((__force u8 *)features_enabled_iomem_addr) &
7852 (1 << (bit_position % BITS_PER_BYTE)) ? true : false;
7853}
7854
7855static inline void pqi_request_firmware_feature(
7856 struct pqi_config_table_firmware_features *firmware_features,
7857 unsigned int bit_position)
7858{
7859 unsigned int byte_index;
7860
7861 byte_index = (bit_position / BITS_PER_BYTE) +
7862 le16_to_cpu(firmware_features->num_elements);
7863
7864 firmware_features->features_supported[byte_index] |=
7865 (1 << (bit_position % BITS_PER_BYTE));
7866}
7867
7868static int pqi_config_table_update(struct pqi_ctrl_info *ctrl_info,
7869 u16 first_section, u16 last_section)
7870{
7871 struct pqi_vendor_general_request request;
7872
7873 memset(&request, 0, sizeof(request));
7874
7875 request.header.iu_type = PQI_REQUEST_IU_VENDOR_GENERAL;
7876 put_unaligned_le16(sizeof(request) - PQI_REQUEST_HEADER_LENGTH,
7877 &request.header.iu_length);
7878 put_unaligned_le16(PQI_VENDOR_GENERAL_CONFIG_TABLE_UPDATE,
7879 &request.function_code);
7880 put_unaligned_le16(first_section,
7881 &request.data.config_table_update.first_section);
7882 put_unaligned_le16(last_section,
7883 &request.data.config_table_update.last_section);
7884
7885 return pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);
7886}
7887
7888static int pqi_enable_firmware_features(struct pqi_ctrl_info *ctrl_info,
7889 struct pqi_config_table_firmware_features *firmware_features,
7890 void __iomem *firmware_features_iomem_addr)
7891{
7892 void *features_requested;
7893 void __iomem *features_requested_iomem_addr;
7894 void __iomem *host_max_known_feature_iomem_addr;
7895
7896 features_requested = firmware_features->features_supported +
7897 le16_to_cpu(firmware_features->num_elements);
7898
7899 features_requested_iomem_addr = firmware_features_iomem_addr +
7900 (features_requested - (void *)firmware_features);
7901
7902 memcpy_toio(features_requested_iomem_addr, features_requested,
7903 le16_to_cpu(firmware_features->num_elements));
7904
7905 if (pqi_is_firmware_feature_supported(firmware_features,
7906 PQI_FIRMWARE_FEATURE_MAX_KNOWN_FEATURE)) {
7907 host_max_known_feature_iomem_addr =
7908 features_requested_iomem_addr +
7909 (le16_to_cpu(firmware_features->num_elements) * 2) +
7910 sizeof(__le16);
7911 writeb(PQI_FIRMWARE_FEATURE_MAXIMUM & 0xFF, host_max_known_feature_iomem_addr);
7912 writeb((PQI_FIRMWARE_FEATURE_MAXIMUM & 0xFF00) >> 8, host_max_known_feature_iomem_addr + 1);
7913 }
7914
7915 return pqi_config_table_update(ctrl_info,
7916 PQI_CONFIG_TABLE_SECTION_FIRMWARE_FEATURES,
7917 PQI_CONFIG_TABLE_SECTION_FIRMWARE_FEATURES);
7918}
7919
7920struct pqi_firmware_feature {
7921 char *feature_name;
7922 unsigned int feature_bit;
7923 bool supported;
7924 bool enabled;
7925 void (*feature_status)(struct pqi_ctrl_info *ctrl_info,
7926 struct pqi_firmware_feature *firmware_feature);
7927};
7928
7929static void pqi_firmware_feature_status(struct pqi_ctrl_info *ctrl_info,
7930 struct pqi_firmware_feature *firmware_feature)
7931{
7932 if (!firmware_feature->supported) {
7933 dev_info(&ctrl_info->pci_dev->dev, "%s not supported by controller\n",
7934 firmware_feature->feature_name);
7935 return;
7936 }
7937
7938 if (firmware_feature->enabled) {
7939 dev_info(&ctrl_info->pci_dev->dev,
7940 "%s enabled\n", firmware_feature->feature_name);
7941 return;
7942 }
7943
7944 dev_err(&ctrl_info->pci_dev->dev, "failed to enable %s\n",
7945 firmware_feature->feature_name);
7946}
7947
7948static void pqi_ctrl_update_feature_flags(struct pqi_ctrl_info *ctrl_info,
7949 struct pqi_firmware_feature *firmware_feature)
7950{
7951 switch (firmware_feature->feature_bit) {
7952 case PQI_FIRMWARE_FEATURE_RAID_1_WRITE_BYPASS:
7953 ctrl_info->enable_r1_writes = firmware_feature->enabled;
7954 break;
7955 case PQI_FIRMWARE_FEATURE_RAID_5_WRITE_BYPASS:
7956 ctrl_info->enable_r5_writes = firmware_feature->enabled;
7957 break;
7958 case PQI_FIRMWARE_FEATURE_RAID_6_WRITE_BYPASS:
7959 ctrl_info->enable_r6_writes = firmware_feature->enabled;
7960 break;
7961 case PQI_FIRMWARE_FEATURE_SOFT_RESET_HANDSHAKE:
7962 ctrl_info->soft_reset_handshake_supported =
7963 firmware_feature->enabled &&
7964 pqi_read_soft_reset_status(ctrl_info);
7965 break;
7966 case PQI_FIRMWARE_FEATURE_RAID_IU_TIMEOUT:
7967 ctrl_info->raid_iu_timeout_supported = firmware_feature->enabled;
7968 break;
7969 case PQI_FIRMWARE_FEATURE_TMF_IU_TIMEOUT:
7970 ctrl_info->tmf_iu_timeout_supported = firmware_feature->enabled;
7971 break;
7972 case PQI_FIRMWARE_FEATURE_FW_TRIAGE:
7973 ctrl_info->firmware_triage_supported = firmware_feature->enabled;
7974 pqi_save_fw_triage_setting(ctrl_info, firmware_feature->enabled);
7975 break;
7976 case PQI_FIRMWARE_FEATURE_RPL_EXTENDED_FORMAT_4_5:
7977 ctrl_info->rpl_extended_format_4_5_supported = firmware_feature->enabled;
7978 break;
7979 case PQI_FIRMWARE_FEATURE_MULTI_LUN_DEVICE_SUPPORT:
7980 ctrl_info->multi_lun_device_supported = firmware_feature->enabled;
7981 break;
7982 case PQI_FIRMWARE_FEATURE_CTRL_LOGGING:
7983 ctrl_info->ctrl_logging_supported = firmware_feature->enabled;
7984 break;
7985 }
7986
7987 pqi_firmware_feature_status(ctrl_info, firmware_feature);
7988}
7989
7990static inline void pqi_firmware_feature_update(struct pqi_ctrl_info *ctrl_info,
7991 struct pqi_firmware_feature *firmware_feature)
7992{
7993 if (firmware_feature->feature_status)
7994 firmware_feature->feature_status(ctrl_info, firmware_feature);
7995}
7996
7997static DEFINE_MUTEX(pqi_firmware_features_mutex);
7998
7999static struct pqi_firmware_feature pqi_firmware_features[] = {
8000 {
8001 .feature_name = "Online Firmware Activation",
8002 .feature_bit = PQI_FIRMWARE_FEATURE_OFA,
8003 .feature_status = pqi_firmware_feature_status,
8004 },
8005 {
8006 .feature_name = "Serial Management Protocol",
8007 .feature_bit = PQI_FIRMWARE_FEATURE_SMP,
8008 .feature_status = pqi_firmware_feature_status,
8009 },
8010 {
8011 .feature_name = "Maximum Known Feature",
8012 .feature_bit = PQI_FIRMWARE_FEATURE_MAX_KNOWN_FEATURE,
8013 .feature_status = pqi_firmware_feature_status,
8014 },
8015 {
8016 .feature_name = "RAID 0 Read Bypass",
8017 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_0_READ_BYPASS,
8018 .feature_status = pqi_firmware_feature_status,
8019 },
8020 {
8021 .feature_name = "RAID 1 Read Bypass",
8022 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_1_READ_BYPASS,
8023 .feature_status = pqi_firmware_feature_status,
8024 },
8025 {
8026 .feature_name = "RAID 5 Read Bypass",
8027 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_5_READ_BYPASS,
8028 .feature_status = pqi_firmware_feature_status,
8029 },
8030 {
8031 .feature_name = "RAID 6 Read Bypass",
8032 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_6_READ_BYPASS,
8033 .feature_status = pqi_firmware_feature_status,
8034 },
8035 {
8036 .feature_name = "RAID 0 Write Bypass",
8037 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_0_WRITE_BYPASS,
8038 .feature_status = pqi_firmware_feature_status,
8039 },
8040 {
8041 .feature_name = "RAID 1 Write Bypass",
8042 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_1_WRITE_BYPASS,
8043 .feature_status = pqi_ctrl_update_feature_flags,
8044 },
8045 {
8046 .feature_name = "RAID 5 Write Bypass",
8047 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_5_WRITE_BYPASS,
8048 .feature_status = pqi_ctrl_update_feature_flags,
8049 },
8050 {
8051 .feature_name = "RAID 6 Write Bypass",
8052 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_6_WRITE_BYPASS,
8053 .feature_status = pqi_ctrl_update_feature_flags,
8054 },
8055 {
8056 .feature_name = "New Soft Reset Handshake",
8057 .feature_bit = PQI_FIRMWARE_FEATURE_SOFT_RESET_HANDSHAKE,
8058 .feature_status = pqi_ctrl_update_feature_flags,
8059 },
8060 {
8061 .feature_name = "RAID IU Timeout",
8062 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_IU_TIMEOUT,
8063 .feature_status = pqi_ctrl_update_feature_flags,
8064 },
8065 {
8066 .feature_name = "TMF IU Timeout",
8067 .feature_bit = PQI_FIRMWARE_FEATURE_TMF_IU_TIMEOUT,
8068 .feature_status = pqi_ctrl_update_feature_flags,
8069 },
8070 {
8071 .feature_name = "RAID Bypass on encrypted logical volumes on NVMe",
8072 .feature_bit = PQI_FIRMWARE_FEATURE_RAID_BYPASS_ON_ENCRYPTED_NVME,
8073 .feature_status = pqi_firmware_feature_status,
8074 },
8075 {
8076 .feature_name = "Firmware Triage",
8077 .feature_bit = PQI_FIRMWARE_FEATURE_FW_TRIAGE,
8078 .feature_status = pqi_ctrl_update_feature_flags,
8079 },
8080 {
8081 .feature_name = "RPL Extended Formats 4 and 5",
8082 .feature_bit = PQI_FIRMWARE_FEATURE_RPL_EXTENDED_FORMAT_4_5,
8083 .feature_status = pqi_ctrl_update_feature_flags,
8084 },
8085 {
8086 .feature_name = "Multi-LUN Target",
8087 .feature_bit = PQI_FIRMWARE_FEATURE_MULTI_LUN_DEVICE_SUPPORT,
8088 .feature_status = pqi_ctrl_update_feature_flags,
8089 },
8090 {
8091 .feature_name = "Controller Data Logging",
8092 .feature_bit = PQI_FIRMWARE_FEATURE_CTRL_LOGGING,
8093 .feature_status = pqi_ctrl_update_feature_flags,
8094 },
8095};
8096
8097static void pqi_process_firmware_features(
8098 struct pqi_config_table_section_info *section_info)
8099{
8100 int rc;
8101 struct pqi_ctrl_info *ctrl_info;
8102 struct pqi_config_table_firmware_features *firmware_features;
8103 void __iomem *firmware_features_iomem_addr;
8104 unsigned int i;
8105 unsigned int num_features_supported;
8106
8107 ctrl_info = section_info->ctrl_info;
8108 firmware_features = section_info->section;
8109 firmware_features_iomem_addr = section_info->section_iomem_addr;
8110
8111 for (i = 0, num_features_supported = 0;
8112 i < ARRAY_SIZE(pqi_firmware_features); i++) {
8113 if (pqi_is_firmware_feature_supported(firmware_features,
8114 pqi_firmware_features[i].feature_bit)) {
8115 pqi_firmware_features[i].supported = true;
8116 num_features_supported++;
8117 } else {
8118 pqi_firmware_feature_update(ctrl_info,
8119 &pqi_firmware_features[i]);
8120 }
8121 }
8122
8123 if (num_features_supported == 0)
8124 return;
8125
8126 for (i = 0; i < ARRAY_SIZE(pqi_firmware_features); i++) {
8127 if (!pqi_firmware_features[i].supported)
8128 continue;
8129 pqi_request_firmware_feature(firmware_features,
8130 pqi_firmware_features[i].feature_bit);
8131 }
8132
8133 rc = pqi_enable_firmware_features(ctrl_info, firmware_features,
8134 firmware_features_iomem_addr);
8135 if (rc) {
8136 dev_err(&ctrl_info->pci_dev->dev,
8137 "failed to enable firmware features in PQI configuration table\n");
8138 for (i = 0; i < ARRAY_SIZE(pqi_firmware_features); i++) {
8139 if (!pqi_firmware_features[i].supported)
8140 continue;
8141 pqi_firmware_feature_update(ctrl_info,
8142 &pqi_firmware_features[i]);
8143 }
8144 return;
8145 }
8146
8147 for (i = 0; i < ARRAY_SIZE(pqi_firmware_features); i++) {
8148 if (!pqi_firmware_features[i].supported)
8149 continue;
8150 if (pqi_is_firmware_feature_enabled(firmware_features,
8151 firmware_features_iomem_addr,
8152 pqi_firmware_features[i].feature_bit)) {
8153 pqi_firmware_features[i].enabled = true;
8154 }
8155 pqi_firmware_feature_update(ctrl_info,
8156 &pqi_firmware_features[i]);
8157 }
8158}
8159
8160static void pqi_init_firmware_features(void)
8161{
8162 unsigned int i;
8163
8164 for (i = 0; i < ARRAY_SIZE(pqi_firmware_features); i++) {
8165 pqi_firmware_features[i].supported = false;
8166 pqi_firmware_features[i].enabled = false;
8167 }
8168}
8169
8170static void pqi_process_firmware_features_section(
8171 struct pqi_config_table_section_info *section_info)
8172{
8173 mutex_lock(&pqi_firmware_features_mutex);
8174 pqi_init_firmware_features();
8175 pqi_process_firmware_features(section_info);
8176 mutex_unlock(&pqi_firmware_features_mutex);
8177}
8178
8179/*
8180 * Reset all controller settings that can be initialized during the processing
8181 * of the PQI Configuration Table.
8182 */
8183
8184static void pqi_ctrl_reset_config(struct pqi_ctrl_info *ctrl_info)
8185{
8186 ctrl_info->heartbeat_counter = NULL;
8187 ctrl_info->soft_reset_status = NULL;
8188 ctrl_info->soft_reset_handshake_supported = false;
8189 ctrl_info->enable_r1_writes = false;
8190 ctrl_info->enable_r5_writes = false;
8191 ctrl_info->enable_r6_writes = false;
8192 ctrl_info->raid_iu_timeout_supported = false;
8193 ctrl_info->tmf_iu_timeout_supported = false;
8194 ctrl_info->firmware_triage_supported = false;
8195 ctrl_info->rpl_extended_format_4_5_supported = false;
8196 ctrl_info->multi_lun_device_supported = false;
8197 ctrl_info->ctrl_logging_supported = false;
8198}
8199
8200static int pqi_process_config_table(struct pqi_ctrl_info *ctrl_info)
8201{
8202 u32 table_length;
8203 u32 section_offset;
8204 bool firmware_feature_section_present;
8205 void __iomem *table_iomem_addr;
8206 struct pqi_config_table *config_table;
8207 struct pqi_config_table_section_header *section;
8208 struct pqi_config_table_section_info section_info;
8209 struct pqi_config_table_section_info feature_section_info = {0};
8210
8211 table_length = ctrl_info->config_table_length;
8212 if (table_length == 0)
8213 return 0;
8214
8215 config_table = kmalloc(table_length, GFP_KERNEL);
8216 if (!config_table) {
8217 dev_err(&ctrl_info->pci_dev->dev,
8218 "failed to allocate memory for PQI configuration table\n");
8219 return -ENOMEM;
8220 }
8221
8222 /*
8223 * Copy the config table contents from I/O memory space into the
8224 * temporary buffer.
8225 */
8226 table_iomem_addr = ctrl_info->iomem_base + ctrl_info->config_table_offset;
8227 memcpy_fromio(config_table, table_iomem_addr, table_length);
8228
8229 firmware_feature_section_present = false;
8230 section_info.ctrl_info = ctrl_info;
8231 section_offset = get_unaligned_le32(&config_table->first_section_offset);
8232
8233 while (section_offset) {
8234 section = (void *)config_table + section_offset;
8235
8236 section_info.section = section;
8237 section_info.section_offset = section_offset;
8238 section_info.section_iomem_addr = table_iomem_addr + section_offset;
8239
8240 switch (get_unaligned_le16(§ion->section_id)) {
8241 case PQI_CONFIG_TABLE_SECTION_FIRMWARE_FEATURES:
8242 firmware_feature_section_present = true;
8243 feature_section_info = section_info;
8244 break;
8245 case PQI_CONFIG_TABLE_SECTION_HEARTBEAT:
8246 if (pqi_disable_heartbeat)
8247 dev_warn(&ctrl_info->pci_dev->dev,
8248 "heartbeat disabled by module parameter\n");
8249 else
8250 ctrl_info->heartbeat_counter =
8251 table_iomem_addr +
8252 section_offset +
8253 offsetof(struct pqi_config_table_heartbeat,
8254 heartbeat_counter);
8255 break;
8256 case PQI_CONFIG_TABLE_SECTION_SOFT_RESET:
8257 ctrl_info->soft_reset_status =
8258 table_iomem_addr +
8259 section_offset +
8260 offsetof(struct pqi_config_table_soft_reset,
8261 soft_reset_status);
8262 break;
8263 }
8264
8265 section_offset = get_unaligned_le16(§ion->next_section_offset);
8266 }
8267
8268 /*
8269 * We process the firmware feature section after all other sections
8270 * have been processed so that the feature bit callbacks can take
8271 * into account the settings configured by other sections.
8272 */
8273 if (firmware_feature_section_present)
8274 pqi_process_firmware_features_section(&feature_section_info);
8275
8276 kfree(config_table);
8277
8278 return 0;
8279}
8280
8281/* Switches the controller from PQI mode back into SIS mode. */
8282
8283static int pqi_revert_to_sis_mode(struct pqi_ctrl_info *ctrl_info)
8284{
8285 int rc;
8286
8287 pqi_change_irq_mode(ctrl_info, IRQ_MODE_NONE);
8288 rc = pqi_reset(ctrl_info);
8289 if (rc)
8290 return rc;
8291 rc = sis_reenable_sis_mode(ctrl_info);
8292 if (rc) {
8293 dev_err(&ctrl_info->pci_dev->dev,
8294 "re-enabling SIS mode failed with error %d\n", rc);
8295 return rc;
8296 }
8297 pqi_save_ctrl_mode(ctrl_info, SIS_MODE);
8298
8299 return 0;
8300}
8301
8302/*
8303 * If the controller isn't already in SIS mode, this function forces it into
8304 * SIS mode.
8305 */
8306
8307static int pqi_force_sis_mode(struct pqi_ctrl_info *ctrl_info)
8308{
8309 if (!sis_is_firmware_running(ctrl_info))
8310 return -ENXIO;
8311
8312 if (pqi_get_ctrl_mode(ctrl_info) == SIS_MODE)
8313 return 0;
8314
8315 if (sis_is_kernel_up(ctrl_info)) {
8316 pqi_save_ctrl_mode(ctrl_info, SIS_MODE);
8317 return 0;
8318 }
8319
8320 return pqi_revert_to_sis_mode(ctrl_info);
8321}
8322
8323static void pqi_perform_lockup_action(void)
8324{
8325 switch (pqi_lockup_action) {
8326 case PANIC:
8327 panic("FATAL: Smart Family Controller lockup detected");
8328 break;
8329 case REBOOT:
8330 emergency_restart();
8331 break;
8332 case NONE:
8333 default:
8334 break;
8335 }
8336}
8337
8338#define PQI_CTRL_LOG_TOTAL_SIZE (4 * 1024 * 1024)
8339#define PQI_CTRL_LOG_MIN_SIZE (PQI_CTRL_LOG_TOTAL_SIZE / PQI_HOST_MAX_SG_DESCRIPTORS)
8340
8341static int pqi_ctrl_init(struct pqi_ctrl_info *ctrl_info)
8342{
8343 int rc;
8344 u32 product_id;
8345
8346 if (reset_devices) {
8347 if (is_kdump_kernel() && pqi_is_fw_triage_supported(ctrl_info)) {
8348 rc = sis_wait_for_fw_triage_completion(ctrl_info);
8349 if (rc)
8350 return rc;
8351 }
8352 if (is_kdump_kernel() && sis_is_ctrl_logging_supported(ctrl_info)) {
8353 sis_notify_kdump(ctrl_info);
8354 rc = sis_wait_for_ctrl_logging_completion(ctrl_info);
8355 if (rc)
8356 return rc;
8357 }
8358 sis_soft_reset(ctrl_info);
8359 ssleep(PQI_POST_RESET_DELAY_SECS);
8360 } else {
8361 rc = pqi_force_sis_mode(ctrl_info);
8362 if (rc)
8363 return rc;
8364 }
8365
8366 /*
8367 * Wait until the controller is ready to start accepting SIS
8368 * commands.
8369 */
8370 rc = sis_wait_for_ctrl_ready(ctrl_info);
8371 if (rc) {
8372 if (reset_devices) {
8373 dev_err(&ctrl_info->pci_dev->dev,
8374 "kdump init failed with error %d\n", rc);
8375 pqi_lockup_action = REBOOT;
8376 pqi_perform_lockup_action();
8377 }
8378 return rc;
8379 }
8380
8381 /*
8382 * Get the controller properties. This allows us to determine
8383 * whether or not it supports PQI mode.
8384 */
8385 rc = sis_get_ctrl_properties(ctrl_info);
8386 if (rc) {
8387 dev_err(&ctrl_info->pci_dev->dev,
8388 "error obtaining controller properties\n");
8389 return rc;
8390 }
8391
8392 rc = sis_get_pqi_capabilities(ctrl_info);
8393 if (rc) {
8394 dev_err(&ctrl_info->pci_dev->dev,
8395 "error obtaining controller capabilities\n");
8396 return rc;
8397 }
8398
8399 product_id = sis_get_product_id(ctrl_info);
8400 ctrl_info->product_id = (u8)product_id;
8401 ctrl_info->product_revision = (u8)(product_id >> 8);
8402
8403 if (is_kdump_kernel()) {
8404 if (ctrl_info->max_outstanding_requests >
8405 PQI_MAX_OUTSTANDING_REQUESTS_KDUMP)
8406 ctrl_info->max_outstanding_requests =
8407 PQI_MAX_OUTSTANDING_REQUESTS_KDUMP;
8408 } else {
8409 if (ctrl_info->max_outstanding_requests >
8410 PQI_MAX_OUTSTANDING_REQUESTS)
8411 ctrl_info->max_outstanding_requests =
8412 PQI_MAX_OUTSTANDING_REQUESTS;
8413 }
8414
8415 pqi_calculate_io_resources(ctrl_info);
8416
8417 rc = pqi_alloc_error_buffer(ctrl_info);
8418 if (rc) {
8419 dev_err(&ctrl_info->pci_dev->dev,
8420 "failed to allocate PQI error buffer\n");
8421 return rc;
8422 }
8423
8424 /*
8425 * If the function we are about to call succeeds, the
8426 * controller will transition from legacy SIS mode
8427 * into PQI mode.
8428 */
8429 rc = sis_init_base_struct_addr(ctrl_info);
8430 if (rc) {
8431 dev_err(&ctrl_info->pci_dev->dev,
8432 "error initializing PQI mode\n");
8433 return rc;
8434 }
8435
8436 /* Wait for the controller to complete the SIS -> PQI transition. */
8437 rc = pqi_wait_for_pqi_mode_ready(ctrl_info);
8438 if (rc) {
8439 dev_err(&ctrl_info->pci_dev->dev,
8440 "transition to PQI mode failed\n");
8441 return rc;
8442 }
8443
8444 /* From here on, we are running in PQI mode. */
8445 ctrl_info->pqi_mode_enabled = true;
8446 pqi_save_ctrl_mode(ctrl_info, PQI_MODE);
8447
8448 rc = pqi_alloc_admin_queues(ctrl_info);
8449 if (rc) {
8450 dev_err(&ctrl_info->pci_dev->dev,
8451 "failed to allocate admin queues\n");
8452 return rc;
8453 }
8454
8455 rc = pqi_create_admin_queues(ctrl_info);
8456 if (rc) {
8457 dev_err(&ctrl_info->pci_dev->dev,
8458 "error creating admin queues\n");
8459 return rc;
8460 }
8461
8462 rc = pqi_report_device_capability(ctrl_info);
8463 if (rc) {
8464 dev_err(&ctrl_info->pci_dev->dev,
8465 "obtaining device capability failed\n");
8466 return rc;
8467 }
8468
8469 rc = pqi_validate_device_capability(ctrl_info);
8470 if (rc)
8471 return rc;
8472
8473 pqi_calculate_queue_resources(ctrl_info);
8474
8475 rc = pqi_enable_msix_interrupts(ctrl_info);
8476 if (rc)
8477 return rc;
8478
8479 if (ctrl_info->num_msix_vectors_enabled < ctrl_info->num_queue_groups) {
8480 ctrl_info->max_msix_vectors =
8481 ctrl_info->num_msix_vectors_enabled;
8482 pqi_calculate_queue_resources(ctrl_info);
8483 }
8484
8485 rc = pqi_alloc_io_resources(ctrl_info);
8486 if (rc)
8487 return rc;
8488
8489 rc = pqi_alloc_operational_queues(ctrl_info);
8490 if (rc) {
8491 dev_err(&ctrl_info->pci_dev->dev,
8492 "failed to allocate operational queues\n");
8493 return rc;
8494 }
8495
8496 pqi_init_operational_queues(ctrl_info);
8497
8498 rc = pqi_create_queues(ctrl_info);
8499 if (rc)
8500 return rc;
8501
8502 rc = pqi_request_irqs(ctrl_info);
8503 if (rc)
8504 return rc;
8505
8506 pqi_change_irq_mode(ctrl_info, IRQ_MODE_MSIX);
8507
8508 ctrl_info->controller_online = true;
8509
8510 rc = pqi_process_config_table(ctrl_info);
8511 if (rc)
8512 return rc;
8513
8514 pqi_start_heartbeat_timer(ctrl_info);
8515
8516 if (ctrl_info->enable_r5_writes || ctrl_info->enable_r6_writes) {
8517 rc = pqi_get_advanced_raid_bypass_config(ctrl_info);
8518 if (rc) { /* Supported features not returned correctly. */
8519 dev_err(&ctrl_info->pci_dev->dev,
8520 "error obtaining advanced RAID bypass configuration\n");
8521 return rc;
8522 }
8523 ctrl_info->ciss_report_log_flags |=
8524 CISS_REPORT_LOG_FLAG_DRIVE_TYPE_MIX;
8525 }
8526
8527 rc = pqi_enable_events(ctrl_info);
8528 if (rc) {
8529 dev_err(&ctrl_info->pci_dev->dev,
8530 "error enabling events\n");
8531 return rc;
8532 }
8533
8534 /* Register with the SCSI subsystem. */
8535 rc = pqi_register_scsi(ctrl_info);
8536 if (rc)
8537 return rc;
8538
8539 if (ctrl_info->ctrl_logging_supported && !is_kdump_kernel()) {
8540 pqi_host_setup_buffer(ctrl_info, &ctrl_info->ctrl_log_memory, PQI_CTRL_LOG_TOTAL_SIZE, PQI_CTRL_LOG_MIN_SIZE);
8541 pqi_host_memory_update(ctrl_info, &ctrl_info->ctrl_log_memory, PQI_VENDOR_GENERAL_CTRL_LOG_MEMORY_UPDATE);
8542 }
8543
8544 rc = pqi_get_ctrl_product_details(ctrl_info);
8545 if (rc) {
8546 dev_err(&ctrl_info->pci_dev->dev,
8547 "error obtaining product details\n");
8548 return rc;
8549 }
8550
8551 rc = pqi_get_ctrl_serial_number(ctrl_info);
8552 if (rc) {
8553 dev_err(&ctrl_info->pci_dev->dev,
8554 "error obtaining ctrl serial number\n");
8555 return rc;
8556 }
8557
8558 rc = pqi_set_diag_rescan(ctrl_info);
8559 if (rc) {
8560 dev_err(&ctrl_info->pci_dev->dev,
8561 "error enabling multi-lun rescan\n");
8562 return rc;
8563 }
8564
8565 rc = pqi_write_driver_version_to_host_wellness(ctrl_info);
8566 if (rc) {
8567 dev_err(&ctrl_info->pci_dev->dev,
8568 "error updating host wellness\n");
8569 return rc;
8570 }
8571
8572 pqi_schedule_update_time_worker(ctrl_info);
8573
8574 pqi_scan_scsi_devices(ctrl_info);
8575
8576 return 0;
8577}
8578
8579static void pqi_reinit_queues(struct pqi_ctrl_info *ctrl_info)
8580{
8581 unsigned int i;
8582 struct pqi_admin_queues *admin_queues;
8583 struct pqi_event_queue *event_queue;
8584
8585 admin_queues = &ctrl_info->admin_queues;
8586 admin_queues->iq_pi_copy = 0;
8587 admin_queues->oq_ci_copy = 0;
8588 writel(0, admin_queues->oq_pi);
8589
8590 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
8591 ctrl_info->queue_groups[i].iq_pi_copy[RAID_PATH] = 0;
8592 ctrl_info->queue_groups[i].iq_pi_copy[AIO_PATH] = 0;
8593 ctrl_info->queue_groups[i].oq_ci_copy = 0;
8594
8595 writel(0, ctrl_info->queue_groups[i].iq_ci[RAID_PATH]);
8596 writel(0, ctrl_info->queue_groups[i].iq_ci[AIO_PATH]);
8597 writel(0, ctrl_info->queue_groups[i].oq_pi);
8598 }
8599
8600 event_queue = &ctrl_info->event_queue;
8601 writel(0, event_queue->oq_pi);
8602 event_queue->oq_ci_copy = 0;
8603}
8604
8605static int pqi_ctrl_init_resume(struct pqi_ctrl_info *ctrl_info)
8606{
8607 int rc;
8608
8609 rc = pqi_force_sis_mode(ctrl_info);
8610 if (rc)
8611 return rc;
8612
8613 /*
8614 * Wait until the controller is ready to start accepting SIS
8615 * commands.
8616 */
8617 rc = sis_wait_for_ctrl_ready_resume(ctrl_info);
8618 if (rc)
8619 return rc;
8620
8621 /*
8622 * Get the controller properties. This allows us to determine
8623 * whether or not it supports PQI mode.
8624 */
8625 rc = sis_get_ctrl_properties(ctrl_info);
8626 if (rc) {
8627 dev_err(&ctrl_info->pci_dev->dev,
8628 "error obtaining controller properties\n");
8629 return rc;
8630 }
8631
8632 rc = sis_get_pqi_capabilities(ctrl_info);
8633 if (rc) {
8634 dev_err(&ctrl_info->pci_dev->dev,
8635 "error obtaining controller capabilities\n");
8636 return rc;
8637 }
8638
8639 /*
8640 * If the function we are about to call succeeds, the
8641 * controller will transition from legacy SIS mode
8642 * into PQI mode.
8643 */
8644 rc = sis_init_base_struct_addr(ctrl_info);
8645 if (rc) {
8646 dev_err(&ctrl_info->pci_dev->dev,
8647 "error initializing PQI mode\n");
8648 return rc;
8649 }
8650
8651 /* Wait for the controller to complete the SIS -> PQI transition. */
8652 rc = pqi_wait_for_pqi_mode_ready(ctrl_info);
8653 if (rc) {
8654 dev_err(&ctrl_info->pci_dev->dev,
8655 "transition to PQI mode failed\n");
8656 return rc;
8657 }
8658
8659 /* From here on, we are running in PQI mode. */
8660 ctrl_info->pqi_mode_enabled = true;
8661 pqi_save_ctrl_mode(ctrl_info, PQI_MODE);
8662
8663 pqi_reinit_queues(ctrl_info);
8664
8665 rc = pqi_create_admin_queues(ctrl_info);
8666 if (rc) {
8667 dev_err(&ctrl_info->pci_dev->dev,
8668 "error creating admin queues\n");
8669 return rc;
8670 }
8671
8672 rc = pqi_create_queues(ctrl_info);
8673 if (rc)
8674 return rc;
8675
8676 pqi_change_irq_mode(ctrl_info, IRQ_MODE_MSIX);
8677
8678 ctrl_info->controller_online = true;
8679 pqi_ctrl_unblock_requests(ctrl_info);
8680
8681 pqi_ctrl_reset_config(ctrl_info);
8682
8683 rc = pqi_process_config_table(ctrl_info);
8684 if (rc)
8685 return rc;
8686
8687 pqi_start_heartbeat_timer(ctrl_info);
8688
8689 if (ctrl_info->enable_r5_writes || ctrl_info->enable_r6_writes) {
8690 rc = pqi_get_advanced_raid_bypass_config(ctrl_info);
8691 if (rc) {
8692 dev_err(&ctrl_info->pci_dev->dev,
8693 "error obtaining advanced RAID bypass configuration\n");
8694 return rc;
8695 }
8696 ctrl_info->ciss_report_log_flags |=
8697 CISS_REPORT_LOG_FLAG_DRIVE_TYPE_MIX;
8698 }
8699
8700 rc = pqi_enable_events(ctrl_info);
8701 if (rc) {
8702 dev_err(&ctrl_info->pci_dev->dev,
8703 "error enabling events\n");
8704 return rc;
8705 }
8706
8707 rc = pqi_get_ctrl_product_details(ctrl_info);
8708 if (rc) {
8709 dev_err(&ctrl_info->pci_dev->dev,
8710 "error obtaining product details\n");
8711 return rc;
8712 }
8713
8714 rc = pqi_set_diag_rescan(ctrl_info);
8715 if (rc) {
8716 dev_err(&ctrl_info->pci_dev->dev,
8717 "error enabling multi-lun rescan\n");
8718 return rc;
8719 }
8720
8721 rc = pqi_write_driver_version_to_host_wellness(ctrl_info);
8722 if (rc) {
8723 dev_err(&ctrl_info->pci_dev->dev,
8724 "error updating host wellness\n");
8725 return rc;
8726 }
8727
8728 if (pqi_ofa_in_progress(ctrl_info)) {
8729 pqi_ctrl_unblock_scan(ctrl_info);
8730 if (ctrl_info->ctrl_logging_supported) {
8731 if (!ctrl_info->ctrl_log_memory.host_memory)
8732 pqi_host_setup_buffer(ctrl_info,
8733 &ctrl_info->ctrl_log_memory,
8734 PQI_CTRL_LOG_TOTAL_SIZE,
8735 PQI_CTRL_LOG_MIN_SIZE);
8736 pqi_host_memory_update(ctrl_info,
8737 &ctrl_info->ctrl_log_memory, PQI_VENDOR_GENERAL_CTRL_LOG_MEMORY_UPDATE);
8738 } else {
8739 if (ctrl_info->ctrl_log_memory.host_memory)
8740 pqi_host_free_buffer(ctrl_info,
8741 &ctrl_info->ctrl_log_memory);
8742 }
8743 }
8744
8745 pqi_scan_scsi_devices(ctrl_info);
8746
8747 return 0;
8748}
8749
8750static inline int pqi_set_pcie_completion_timeout(struct pci_dev *pci_dev, u16 timeout)
8751{
8752 int rc;
8753
8754 rc = pcie_capability_clear_and_set_word(pci_dev, PCI_EXP_DEVCTL2,
8755 PCI_EXP_DEVCTL2_COMP_TIMEOUT, timeout);
8756
8757 return pcibios_err_to_errno(rc);
8758}
8759
8760static int pqi_pci_init(struct pqi_ctrl_info *ctrl_info)
8761{
8762 int rc;
8763 u64 mask;
8764
8765 rc = pci_enable_device(ctrl_info->pci_dev);
8766 if (rc) {
8767 dev_err(&ctrl_info->pci_dev->dev,
8768 "failed to enable PCI device\n");
8769 return rc;
8770 }
8771
8772 if (sizeof(dma_addr_t) > 4)
8773 mask = DMA_BIT_MASK(64);
8774 else
8775 mask = DMA_BIT_MASK(32);
8776
8777 rc = dma_set_mask_and_coherent(&ctrl_info->pci_dev->dev, mask);
8778 if (rc) {
8779 dev_err(&ctrl_info->pci_dev->dev, "failed to set DMA mask\n");
8780 goto disable_device;
8781 }
8782
8783 rc = pci_request_regions(ctrl_info->pci_dev, DRIVER_NAME_SHORT);
8784 if (rc) {
8785 dev_err(&ctrl_info->pci_dev->dev,
8786 "failed to obtain PCI resources\n");
8787 goto disable_device;
8788 }
8789
8790 ctrl_info->iomem_base = ioremap(pci_resource_start(
8791 ctrl_info->pci_dev, 0),
8792 pci_resource_len(ctrl_info->pci_dev, 0));
8793 if (!ctrl_info->iomem_base) {
8794 dev_err(&ctrl_info->pci_dev->dev,
8795 "failed to map memory for controller registers\n");
8796 rc = -ENOMEM;
8797 goto release_regions;
8798 }
8799
8800#define PCI_EXP_COMP_TIMEOUT_65_TO_210_MS 0x6
8801
8802 /* Increase the PCIe completion timeout. */
8803 rc = pqi_set_pcie_completion_timeout(ctrl_info->pci_dev,
8804 PCI_EXP_COMP_TIMEOUT_65_TO_210_MS);
8805 if (rc) {
8806 dev_err(&ctrl_info->pci_dev->dev,
8807 "failed to set PCIe completion timeout\n");
8808 goto release_regions;
8809 }
8810
8811 /* Enable bus mastering. */
8812 pci_set_master(ctrl_info->pci_dev);
8813
8814 ctrl_info->registers = ctrl_info->iomem_base;
8815 ctrl_info->pqi_registers = &ctrl_info->registers->pqi_registers;
8816
8817 pci_set_drvdata(ctrl_info->pci_dev, ctrl_info);
8818
8819 return 0;
8820
8821release_regions:
8822 pci_release_regions(ctrl_info->pci_dev);
8823disable_device:
8824 pci_disable_device(ctrl_info->pci_dev);
8825
8826 return rc;
8827}
8828
8829static void pqi_cleanup_pci_init(struct pqi_ctrl_info *ctrl_info)
8830{
8831 iounmap(ctrl_info->iomem_base);
8832 pci_release_regions(ctrl_info->pci_dev);
8833 if (pci_is_enabled(ctrl_info->pci_dev))
8834 pci_disable_device(ctrl_info->pci_dev);
8835 pci_set_drvdata(ctrl_info->pci_dev, NULL);
8836}
8837
8838static struct pqi_ctrl_info *pqi_alloc_ctrl_info(int numa_node)
8839{
8840 struct pqi_ctrl_info *ctrl_info;
8841
8842 ctrl_info = kzalloc_node(sizeof(struct pqi_ctrl_info),
8843 GFP_KERNEL, numa_node);
8844 if (!ctrl_info)
8845 return NULL;
8846
8847 mutex_init(&ctrl_info->scan_mutex);
8848 mutex_init(&ctrl_info->lun_reset_mutex);
8849 mutex_init(&ctrl_info->ofa_mutex);
8850
8851 INIT_LIST_HEAD(&ctrl_info->scsi_device_list);
8852 spin_lock_init(&ctrl_info->scsi_device_list_lock);
8853
8854 INIT_WORK(&ctrl_info->event_work, pqi_event_worker);
8855 atomic_set(&ctrl_info->num_interrupts, 0);
8856
8857 INIT_DELAYED_WORK(&ctrl_info->rescan_work, pqi_rescan_worker);
8858 INIT_DELAYED_WORK(&ctrl_info->update_time_work, pqi_update_time_worker);
8859
8860 timer_setup(&ctrl_info->heartbeat_timer, pqi_heartbeat_timer_handler, 0);
8861 INIT_WORK(&ctrl_info->ctrl_offline_work, pqi_ctrl_offline_worker);
8862
8863 INIT_WORK(&ctrl_info->ofa_memory_alloc_work, pqi_ofa_memory_alloc_worker);
8864 INIT_WORK(&ctrl_info->ofa_quiesce_work, pqi_ofa_quiesce_worker);
8865
8866 sema_init(&ctrl_info->sync_request_sem,
8867 PQI_RESERVED_IO_SLOTS_SYNCHRONOUS_REQUESTS);
8868 init_waitqueue_head(&ctrl_info->block_requests_wait);
8869
8870 ctrl_info->ctrl_id = atomic_inc_return(&pqi_controller_count) - 1;
8871 ctrl_info->irq_mode = IRQ_MODE_NONE;
8872 ctrl_info->max_msix_vectors = PQI_MAX_MSIX_VECTORS;
8873
8874 ctrl_info->ciss_report_log_flags = CISS_REPORT_LOG_FLAG_UNIQUE_LUN_ID;
8875 ctrl_info->max_transfer_encrypted_sas_sata =
8876 PQI_DEFAULT_MAX_TRANSFER_ENCRYPTED_SAS_SATA;
8877 ctrl_info->max_transfer_encrypted_nvme =
8878 PQI_DEFAULT_MAX_TRANSFER_ENCRYPTED_NVME;
8879 ctrl_info->max_write_raid_5_6 = PQI_DEFAULT_MAX_WRITE_RAID_5_6;
8880 ctrl_info->max_write_raid_1_10_2drive = ~0;
8881 ctrl_info->max_write_raid_1_10_3drive = ~0;
8882 ctrl_info->disable_managed_interrupts = pqi_disable_managed_interrupts;
8883
8884 return ctrl_info;
8885}
8886
8887static inline void pqi_free_ctrl_info(struct pqi_ctrl_info *ctrl_info)
8888{
8889 kfree(ctrl_info);
8890}
8891
8892static void pqi_free_interrupts(struct pqi_ctrl_info *ctrl_info)
8893{
8894 pqi_free_irqs(ctrl_info);
8895 pqi_disable_msix_interrupts(ctrl_info);
8896}
8897
8898static void pqi_free_ctrl_resources(struct pqi_ctrl_info *ctrl_info)
8899{
8900 pqi_free_interrupts(ctrl_info);
8901 if (ctrl_info->queue_memory_base)
8902 dma_free_coherent(&ctrl_info->pci_dev->dev,
8903 ctrl_info->queue_memory_length,
8904 ctrl_info->queue_memory_base,
8905 ctrl_info->queue_memory_base_dma_handle);
8906 if (ctrl_info->admin_queue_memory_base)
8907 dma_free_coherent(&ctrl_info->pci_dev->dev,
8908 ctrl_info->admin_queue_memory_length,
8909 ctrl_info->admin_queue_memory_base,
8910 ctrl_info->admin_queue_memory_base_dma_handle);
8911 pqi_free_all_io_requests(ctrl_info);
8912 if (ctrl_info->error_buffer)
8913 dma_free_coherent(&ctrl_info->pci_dev->dev,
8914 ctrl_info->error_buffer_length,
8915 ctrl_info->error_buffer,
8916 ctrl_info->error_buffer_dma_handle);
8917 if (ctrl_info->iomem_base)
8918 pqi_cleanup_pci_init(ctrl_info);
8919 pqi_free_ctrl_info(ctrl_info);
8920}
8921
8922static void pqi_remove_ctrl(struct pqi_ctrl_info *ctrl_info)
8923{
8924 ctrl_info->controller_online = false;
8925 pqi_stop_heartbeat_timer(ctrl_info);
8926 pqi_ctrl_block_requests(ctrl_info);
8927 pqi_cancel_rescan_worker(ctrl_info);
8928 pqi_cancel_update_time_worker(ctrl_info);
8929 if (ctrl_info->ctrl_removal_state == PQI_CTRL_SURPRISE_REMOVAL) {
8930 pqi_fail_all_outstanding_requests(ctrl_info);
8931 ctrl_info->pqi_mode_enabled = false;
8932 }
8933 pqi_host_free_buffer(ctrl_info, &ctrl_info->ctrl_log_memory);
8934 pqi_unregister_scsi(ctrl_info);
8935 if (ctrl_info->pqi_mode_enabled)
8936 pqi_revert_to_sis_mode(ctrl_info);
8937 pqi_free_ctrl_resources(ctrl_info);
8938}
8939
8940static void pqi_ofa_ctrl_quiesce(struct pqi_ctrl_info *ctrl_info)
8941{
8942 pqi_ctrl_block_scan(ctrl_info);
8943 pqi_scsi_block_requests(ctrl_info);
8944 pqi_ctrl_block_device_reset(ctrl_info);
8945 pqi_ctrl_block_requests(ctrl_info);
8946 pqi_ctrl_wait_until_quiesced(ctrl_info);
8947 pqi_stop_heartbeat_timer(ctrl_info);
8948}
8949
8950static void pqi_ofa_ctrl_unquiesce(struct pqi_ctrl_info *ctrl_info)
8951{
8952 pqi_start_heartbeat_timer(ctrl_info);
8953 pqi_ctrl_unblock_requests(ctrl_info);
8954 pqi_ctrl_unblock_device_reset(ctrl_info);
8955 pqi_scsi_unblock_requests(ctrl_info);
8956 pqi_ctrl_unblock_scan(ctrl_info);
8957}
8958
8959static int pqi_ofa_ctrl_restart(struct pqi_ctrl_info *ctrl_info, unsigned int delay_secs)
8960{
8961 ssleep(delay_secs);
8962
8963 return pqi_ctrl_init_resume(ctrl_info);
8964}
8965
8966static int pqi_host_alloc_mem(struct pqi_ctrl_info *ctrl_info,
8967 struct pqi_host_memory_descriptor *host_memory_descriptor,
8968 u32 total_size, u32 chunk_size)
8969{
8970 int i;
8971 u32 sg_count;
8972 struct device *dev;
8973 struct pqi_host_memory *host_memory;
8974 struct pqi_sg_descriptor *mem_descriptor;
8975 dma_addr_t dma_handle;
8976
8977 sg_count = DIV_ROUND_UP(total_size, chunk_size);
8978 if (sg_count == 0 || sg_count > PQI_HOST_MAX_SG_DESCRIPTORS)
8979 goto out;
8980
8981 host_memory_descriptor->host_chunk_virt_address =
8982 kmalloc_array(sg_count, sizeof(void *), GFP_KERNEL);
8983 if (!host_memory_descriptor->host_chunk_virt_address)
8984 goto out;
8985
8986 dev = &ctrl_info->pci_dev->dev;
8987 host_memory = host_memory_descriptor->host_memory;
8988
8989 for (i = 0; i < sg_count; i++) {
8990 host_memory_descriptor->host_chunk_virt_address[i] = dma_alloc_coherent(dev, chunk_size, &dma_handle, GFP_KERNEL);
8991 if (!host_memory_descriptor->host_chunk_virt_address[i])
8992 goto out_free_chunks;
8993 mem_descriptor = &host_memory->sg_descriptor[i];
8994 put_unaligned_le64((u64)dma_handle, &mem_descriptor->address);
8995 put_unaligned_le32(chunk_size, &mem_descriptor->length);
8996 }
8997
8998 put_unaligned_le32(CISS_SG_LAST, &mem_descriptor->flags);
8999 put_unaligned_le16(sg_count, &host_memory->num_memory_descriptors);
9000 put_unaligned_le32(sg_count * chunk_size, &host_memory->bytes_allocated);
9001
9002 return 0;
9003
9004out_free_chunks:
9005 while (--i >= 0) {
9006 mem_descriptor = &host_memory->sg_descriptor[i];
9007 dma_free_coherent(dev, chunk_size,
9008 host_memory_descriptor->host_chunk_virt_address[i],
9009 get_unaligned_le64(&mem_descriptor->address));
9010 }
9011 kfree(host_memory_descriptor->host_chunk_virt_address);
9012out:
9013 return -ENOMEM;
9014}
9015
9016static int pqi_host_alloc_buffer(struct pqi_ctrl_info *ctrl_info,
9017 struct pqi_host_memory_descriptor *host_memory_descriptor,
9018 u32 total_required_size, u32 min_required_size)
9019{
9020 u32 chunk_size;
9021 u32 min_chunk_size;
9022
9023 if (total_required_size == 0 || min_required_size == 0)
9024 return 0;
9025
9026 total_required_size = PAGE_ALIGN(total_required_size);
9027 min_required_size = PAGE_ALIGN(min_required_size);
9028 min_chunk_size = DIV_ROUND_UP(total_required_size, PQI_HOST_MAX_SG_DESCRIPTORS);
9029 min_chunk_size = PAGE_ALIGN(min_chunk_size);
9030
9031 while (total_required_size >= min_required_size) {
9032 for (chunk_size = total_required_size; chunk_size >= min_chunk_size;) {
9033 if (pqi_host_alloc_mem(ctrl_info,
9034 host_memory_descriptor, total_required_size,
9035 chunk_size) == 0)
9036 return 0;
9037 chunk_size /= 2;
9038 chunk_size = PAGE_ALIGN(chunk_size);
9039 }
9040 total_required_size /= 2;
9041 total_required_size = PAGE_ALIGN(total_required_size);
9042 }
9043
9044 return -ENOMEM;
9045}
9046
9047static void pqi_host_setup_buffer(struct pqi_ctrl_info *ctrl_info,
9048 struct pqi_host_memory_descriptor *host_memory_descriptor,
9049 u32 total_size, u32 min_size)
9050{
9051 struct device *dev;
9052 struct pqi_host_memory *host_memory;
9053
9054 dev = &ctrl_info->pci_dev->dev;
9055
9056 host_memory = dma_alloc_coherent(dev, sizeof(*host_memory),
9057 &host_memory_descriptor->host_memory_dma_handle, GFP_KERNEL);
9058 if (!host_memory)
9059 return;
9060
9061 host_memory_descriptor->host_memory = host_memory;
9062
9063 if (pqi_host_alloc_buffer(ctrl_info, host_memory_descriptor,
9064 total_size, min_size) < 0) {
9065 dev_err(dev, "failed to allocate firmware usable host buffer\n");
9066 dma_free_coherent(dev, sizeof(*host_memory), host_memory,
9067 host_memory_descriptor->host_memory_dma_handle);
9068 host_memory_descriptor->host_memory = NULL;
9069 return;
9070 }
9071}
9072
9073static void pqi_host_free_buffer(struct pqi_ctrl_info *ctrl_info,
9074 struct pqi_host_memory_descriptor *host_memory_descriptor)
9075{
9076 unsigned int i;
9077 struct device *dev;
9078 struct pqi_host_memory *host_memory;
9079 struct pqi_sg_descriptor *mem_descriptor;
9080 unsigned int num_memory_descriptors;
9081
9082 host_memory = host_memory_descriptor->host_memory;
9083 if (!host_memory)
9084 return;
9085
9086 dev = &ctrl_info->pci_dev->dev;
9087
9088 if (get_unaligned_le32(&host_memory->bytes_allocated) == 0)
9089 goto out;
9090
9091 mem_descriptor = host_memory->sg_descriptor;
9092 num_memory_descriptors = get_unaligned_le16(&host_memory->num_memory_descriptors);
9093
9094 for (i = 0; i < num_memory_descriptors; i++) {
9095 dma_free_coherent(dev,
9096 get_unaligned_le32(&mem_descriptor[i].length),
9097 host_memory_descriptor->host_chunk_virt_address[i],
9098 get_unaligned_le64(&mem_descriptor[i].address));
9099 }
9100 kfree(host_memory_descriptor->host_chunk_virt_address);
9101
9102out:
9103 dma_free_coherent(dev, sizeof(*host_memory), host_memory,
9104 host_memory_descriptor->host_memory_dma_handle);
9105 host_memory_descriptor->host_memory = NULL;
9106}
9107
9108static int pqi_host_memory_update(struct pqi_ctrl_info *ctrl_info,
9109 struct pqi_host_memory_descriptor *host_memory_descriptor,
9110 u16 function_code)
9111{
9112 u32 buffer_length;
9113 struct pqi_vendor_general_request request;
9114 struct pqi_host_memory *host_memory;
9115
9116 memset(&request, 0, sizeof(request));
9117
9118 request.header.iu_type = PQI_REQUEST_IU_VENDOR_GENERAL;
9119 put_unaligned_le16(sizeof(request) - PQI_REQUEST_HEADER_LENGTH, &request.header.iu_length);
9120 put_unaligned_le16(function_code, &request.function_code);
9121
9122 host_memory = host_memory_descriptor->host_memory;
9123
9124 if (host_memory) {
9125 buffer_length = offsetof(struct pqi_host_memory, sg_descriptor) + get_unaligned_le16(&host_memory->num_memory_descriptors) * sizeof(struct pqi_sg_descriptor);
9126 put_unaligned_le64((u64)host_memory_descriptor->host_memory_dma_handle, &request.data.host_memory_allocation.buffer_address);
9127 put_unaligned_le32(buffer_length, &request.data.host_memory_allocation.buffer_length);
9128
9129 if (function_code == PQI_VENDOR_GENERAL_OFA_MEMORY_UPDATE) {
9130 put_unaligned_le16(PQI_OFA_VERSION, &host_memory->version);
9131 memcpy(&host_memory->signature, PQI_OFA_SIGNATURE, sizeof(host_memory->signature));
9132 } else if (function_code == PQI_VENDOR_GENERAL_CTRL_LOG_MEMORY_UPDATE) {
9133 put_unaligned_le16(PQI_CTRL_LOG_VERSION, &host_memory->version);
9134 memcpy(&host_memory->signature, PQI_CTRL_LOG_SIGNATURE, sizeof(host_memory->signature));
9135 }
9136 }
9137
9138 return pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);
9139}
9140
9141static struct pqi_raid_error_info pqi_ctrl_offline_raid_error_info = {
9142 .data_out_result = PQI_DATA_IN_OUT_HARDWARE_ERROR,
9143 .status = SAM_STAT_CHECK_CONDITION,
9144};
9145
9146static void pqi_fail_all_outstanding_requests(struct pqi_ctrl_info *ctrl_info)
9147{
9148 unsigned int i;
9149 struct pqi_io_request *io_request;
9150 struct scsi_cmnd *scmd;
9151 struct scsi_device *sdev;
9152
9153 for (i = 0; i < ctrl_info->max_io_slots; i++) {
9154 io_request = &ctrl_info->io_request_pool[i];
9155 if (atomic_read(&io_request->refcount) == 0)
9156 continue;
9157
9158 scmd = io_request->scmd;
9159 if (scmd) {
9160 sdev = scmd->device;
9161 if (!sdev || !scsi_device_online(sdev)) {
9162 pqi_free_io_request(io_request);
9163 continue;
9164 } else {
9165 set_host_byte(scmd, DID_NO_CONNECT);
9166 }
9167 } else {
9168 io_request->status = -ENXIO;
9169 io_request->error_info =
9170 &pqi_ctrl_offline_raid_error_info;
9171 }
9172
9173 io_request->io_complete_callback(io_request,
9174 io_request->context);
9175 }
9176}
9177
9178static void pqi_take_ctrl_offline_deferred(struct pqi_ctrl_info *ctrl_info)
9179{
9180 pqi_perform_lockup_action();
9181 pqi_stop_heartbeat_timer(ctrl_info);
9182 pqi_free_interrupts(ctrl_info);
9183 pqi_cancel_rescan_worker(ctrl_info);
9184 pqi_cancel_update_time_worker(ctrl_info);
9185 pqi_ctrl_wait_until_quiesced(ctrl_info);
9186 pqi_fail_all_outstanding_requests(ctrl_info);
9187 pqi_ctrl_unblock_requests(ctrl_info);
9188 pqi_take_ctrl_devices_offline(ctrl_info);
9189}
9190
9191static void pqi_ctrl_offline_worker(struct work_struct *work)
9192{
9193 struct pqi_ctrl_info *ctrl_info;
9194
9195 ctrl_info = container_of(work, struct pqi_ctrl_info, ctrl_offline_work);
9196 pqi_take_ctrl_offline_deferred(ctrl_info);
9197}
9198
9199static char *pqi_ctrl_shutdown_reason_to_string(enum pqi_ctrl_shutdown_reason ctrl_shutdown_reason)
9200{
9201 char *string;
9202
9203 switch (ctrl_shutdown_reason) {
9204 case PQI_IQ_NOT_DRAINED_TIMEOUT:
9205 string = "inbound queue not drained timeout";
9206 break;
9207 case PQI_LUN_RESET_TIMEOUT:
9208 string = "LUN reset timeout";
9209 break;
9210 case PQI_IO_PENDING_POST_LUN_RESET_TIMEOUT:
9211 string = "I/O pending timeout after LUN reset";
9212 break;
9213 case PQI_NO_HEARTBEAT:
9214 string = "no controller heartbeat detected";
9215 break;
9216 case PQI_FIRMWARE_KERNEL_NOT_UP:
9217 string = "firmware kernel not ready";
9218 break;
9219 case PQI_OFA_RESPONSE_TIMEOUT:
9220 string = "OFA response timeout";
9221 break;
9222 case PQI_INVALID_REQ_ID:
9223 string = "invalid request ID";
9224 break;
9225 case PQI_UNMATCHED_REQ_ID:
9226 string = "unmatched request ID";
9227 break;
9228 case PQI_IO_PI_OUT_OF_RANGE:
9229 string = "I/O queue producer index out of range";
9230 break;
9231 case PQI_EVENT_PI_OUT_OF_RANGE:
9232 string = "event queue producer index out of range";
9233 break;
9234 case PQI_UNEXPECTED_IU_TYPE:
9235 string = "unexpected IU type";
9236 break;
9237 default:
9238 string = "unknown reason";
9239 break;
9240 }
9241
9242 return string;
9243}
9244
9245static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info,
9246 enum pqi_ctrl_shutdown_reason ctrl_shutdown_reason)
9247{
9248 if (!ctrl_info->controller_online)
9249 return;
9250
9251 ctrl_info->controller_online = false;
9252 ctrl_info->pqi_mode_enabled = false;
9253 pqi_ctrl_block_requests(ctrl_info);
9254 if (!pqi_disable_ctrl_shutdown)
9255 sis_shutdown_ctrl(ctrl_info, ctrl_shutdown_reason);
9256 pci_disable_device(ctrl_info->pci_dev);
9257 dev_err(&ctrl_info->pci_dev->dev,
9258 "controller offline: reason code 0x%x (%s)\n",
9259 ctrl_shutdown_reason, pqi_ctrl_shutdown_reason_to_string(ctrl_shutdown_reason));
9260 schedule_work(&ctrl_info->ctrl_offline_work);
9261}
9262
9263static void pqi_take_ctrl_devices_offline(struct pqi_ctrl_info *ctrl_info)
9264{
9265 int rc;
9266 unsigned long flags;
9267 struct pqi_scsi_dev *device;
9268
9269 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
9270 list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry) {
9271 rc = list_is_last(&device->scsi_device_list_entry, &ctrl_info->scsi_device_list);
9272 if (rc)
9273 continue;
9274
9275 /*
9276 * Is the sdev pointer NULL?
9277 */
9278 if (device->sdev)
9279 scsi_device_set_state(device->sdev, SDEV_OFFLINE);
9280 }
9281 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
9282}
9283
9284static void pqi_print_ctrl_info(struct pci_dev *pci_dev,
9285 const struct pci_device_id *id)
9286{
9287 char *ctrl_description;
9288
9289 if (id->driver_data)
9290 ctrl_description = (char *)id->driver_data;
9291 else
9292 ctrl_description = "Microchip Smart Family Controller";
9293
9294 dev_info(&pci_dev->dev, "%s found\n", ctrl_description);
9295}
9296
9297static int pqi_pci_probe(struct pci_dev *pci_dev,
9298 const struct pci_device_id *id)
9299{
9300 int rc;
9301 int node;
9302 struct pqi_ctrl_info *ctrl_info;
9303
9304 pqi_print_ctrl_info(pci_dev, id);
9305
9306 if (pqi_disable_device_id_wildcards &&
9307 id->subvendor == PCI_ANY_ID &&
9308 id->subdevice == PCI_ANY_ID) {
9309 dev_warn(&pci_dev->dev,
9310 "controller not probed because device ID wildcards are disabled\n");
9311 return -ENODEV;
9312 }
9313
9314 if (id->subvendor == PCI_ANY_ID || id->subdevice == PCI_ANY_ID)
9315 dev_warn(&pci_dev->dev,
9316 "controller device ID matched using wildcards\n");
9317
9318 node = dev_to_node(&pci_dev->dev);
9319 if (node == NUMA_NO_NODE) {
9320 node = cpu_to_node(0);
9321 if (node == NUMA_NO_NODE)
9322 node = 0;
9323 set_dev_node(&pci_dev->dev, node);
9324 }
9325
9326 ctrl_info = pqi_alloc_ctrl_info(node);
9327 if (!ctrl_info) {
9328 dev_err(&pci_dev->dev,
9329 "failed to allocate controller info block\n");
9330 return -ENOMEM;
9331 }
9332 ctrl_info->numa_node = node;
9333
9334 ctrl_info->pci_dev = pci_dev;
9335
9336 rc = pqi_pci_init(ctrl_info);
9337 if (rc)
9338 goto error;
9339
9340 rc = pqi_ctrl_init(ctrl_info);
9341 if (rc)
9342 goto error;
9343
9344 return 0;
9345
9346error:
9347 pqi_remove_ctrl(ctrl_info);
9348
9349 return rc;
9350}
9351
9352static void pqi_pci_remove(struct pci_dev *pci_dev)
9353{
9354 struct pqi_ctrl_info *ctrl_info;
9355 u16 vendor_id;
9356 int rc;
9357
9358 ctrl_info = pci_get_drvdata(pci_dev);
9359 if (!ctrl_info)
9360 return;
9361
9362 pci_read_config_word(ctrl_info->pci_dev, PCI_SUBSYSTEM_VENDOR_ID, &vendor_id);
9363 if (vendor_id == 0xffff)
9364 ctrl_info->ctrl_removal_state = PQI_CTRL_SURPRISE_REMOVAL;
9365 else
9366 ctrl_info->ctrl_removal_state = PQI_CTRL_GRACEFUL_REMOVAL;
9367
9368 if (ctrl_info->ctrl_removal_state == PQI_CTRL_GRACEFUL_REMOVAL) {
9369 rc = pqi_flush_cache(ctrl_info, RESTART);
9370 if (rc)
9371 dev_err(&pci_dev->dev,
9372 "unable to flush controller cache during remove\n");
9373 }
9374
9375 pqi_remove_ctrl(ctrl_info);
9376}
9377
9378static void pqi_crash_if_pending_command(struct pqi_ctrl_info *ctrl_info)
9379{
9380 unsigned int i;
9381 struct pqi_io_request *io_request;
9382 struct scsi_cmnd *scmd;
9383
9384 for (i = 0; i < ctrl_info->max_io_slots; i++) {
9385 io_request = &ctrl_info->io_request_pool[i];
9386 if (atomic_read(&io_request->refcount) == 0)
9387 continue;
9388 scmd = io_request->scmd;
9389 WARN_ON(scmd != NULL); /* IO command from SML */
9390 WARN_ON(scmd == NULL); /* Non-IO cmd or driver initiated*/
9391 }
9392}
9393
9394static void pqi_shutdown(struct pci_dev *pci_dev)
9395{
9396 int rc;
9397 struct pqi_ctrl_info *ctrl_info;
9398 enum bmic_flush_cache_shutdown_event shutdown_event;
9399
9400 ctrl_info = pci_get_drvdata(pci_dev);
9401 if (!ctrl_info) {
9402 dev_err(&pci_dev->dev,
9403 "cache could not be flushed\n");
9404 return;
9405 }
9406
9407 pqi_wait_until_ofa_finished(ctrl_info);
9408
9409 pqi_scsi_block_requests(ctrl_info);
9410 pqi_ctrl_block_device_reset(ctrl_info);
9411 pqi_ctrl_block_requests(ctrl_info);
9412 pqi_ctrl_wait_until_quiesced(ctrl_info);
9413
9414 if (system_state == SYSTEM_RESTART)
9415 shutdown_event = RESTART;
9416 else
9417 shutdown_event = SHUTDOWN;
9418
9419 /*
9420 * Write all data in the controller's battery-backed cache to
9421 * storage.
9422 */
9423 rc = pqi_flush_cache(ctrl_info, shutdown_event);
9424 if (rc)
9425 dev_err(&pci_dev->dev,
9426 "unable to flush controller cache during shutdown\n");
9427
9428 pqi_crash_if_pending_command(ctrl_info);
9429 pqi_reset(ctrl_info);
9430 pqi_ctrl_unblock_device_reset(ctrl_info);
9431}
9432
9433static void pqi_process_lockup_action_param(void)
9434{
9435 unsigned int i;
9436
9437 if (!pqi_lockup_action_param)
9438 return;
9439
9440 for (i = 0; i < ARRAY_SIZE(pqi_lockup_actions); i++) {
9441 if (strcmp(pqi_lockup_action_param,
9442 pqi_lockup_actions[i].name) == 0) {
9443 pqi_lockup_action = pqi_lockup_actions[i].action;
9444 return;
9445 }
9446 }
9447
9448 pr_warn("%s: invalid lockup action setting \"%s\" - supported settings: none, reboot, panic\n",
9449 DRIVER_NAME_SHORT, pqi_lockup_action_param);
9450}
9451
9452#define PQI_CTRL_READY_TIMEOUT_PARAM_MIN_SECS 30
9453#define PQI_CTRL_READY_TIMEOUT_PARAM_MAX_SECS (30 * 60)
9454
9455static void pqi_process_ctrl_ready_timeout_param(void)
9456{
9457 if (pqi_ctrl_ready_timeout_secs == 0)
9458 return;
9459
9460 if (pqi_ctrl_ready_timeout_secs < PQI_CTRL_READY_TIMEOUT_PARAM_MIN_SECS) {
9461 pr_warn("%s: ctrl_ready_timeout parm of %u second(s) is less than minimum timeout of %d seconds - setting timeout to %d seconds\n",
9462 DRIVER_NAME_SHORT, pqi_ctrl_ready_timeout_secs, PQI_CTRL_READY_TIMEOUT_PARAM_MIN_SECS, PQI_CTRL_READY_TIMEOUT_PARAM_MIN_SECS);
9463 pqi_ctrl_ready_timeout_secs = PQI_CTRL_READY_TIMEOUT_PARAM_MIN_SECS;
9464 } else if (pqi_ctrl_ready_timeout_secs > PQI_CTRL_READY_TIMEOUT_PARAM_MAX_SECS) {
9465 pr_warn("%s: ctrl_ready_timeout parm of %u seconds is greater than maximum timeout of %d seconds - setting timeout to %d seconds\n",
9466 DRIVER_NAME_SHORT, pqi_ctrl_ready_timeout_secs, PQI_CTRL_READY_TIMEOUT_PARAM_MAX_SECS, PQI_CTRL_READY_TIMEOUT_PARAM_MAX_SECS);
9467 pqi_ctrl_ready_timeout_secs = PQI_CTRL_READY_TIMEOUT_PARAM_MAX_SECS;
9468 }
9469
9470 sis_ctrl_ready_timeout_secs = pqi_ctrl_ready_timeout_secs;
9471}
9472
9473static void pqi_process_module_params(void)
9474{
9475 pqi_process_lockup_action_param();
9476 pqi_process_ctrl_ready_timeout_param();
9477}
9478
9479#if defined(CONFIG_PM)
9480
9481static inline enum bmic_flush_cache_shutdown_event pqi_get_flush_cache_shutdown_event(struct pci_dev *pci_dev)
9482{
9483 if (pci_dev->subsystem_vendor == PCI_VENDOR_ID_ADAPTEC2 && pci_dev->subsystem_device == 0x1304)
9484 return RESTART;
9485
9486 return SUSPEND;
9487}
9488
9489static int pqi_suspend_or_freeze(struct device *dev, bool suspend)
9490{
9491 struct pci_dev *pci_dev;
9492 struct pqi_ctrl_info *ctrl_info;
9493
9494 pci_dev = to_pci_dev(dev);
9495 ctrl_info = pci_get_drvdata(pci_dev);
9496
9497 pqi_wait_until_ofa_finished(ctrl_info);
9498
9499 pqi_ctrl_block_scan(ctrl_info);
9500 pqi_scsi_block_requests(ctrl_info);
9501 pqi_ctrl_block_device_reset(ctrl_info);
9502 pqi_ctrl_block_requests(ctrl_info);
9503 pqi_ctrl_wait_until_quiesced(ctrl_info);
9504
9505 if (suspend) {
9506 enum bmic_flush_cache_shutdown_event shutdown_event;
9507
9508 shutdown_event = pqi_get_flush_cache_shutdown_event(pci_dev);
9509 pqi_flush_cache(ctrl_info, shutdown_event);
9510 }
9511
9512 pqi_stop_heartbeat_timer(ctrl_info);
9513 pqi_crash_if_pending_command(ctrl_info);
9514 pqi_free_irqs(ctrl_info);
9515
9516 ctrl_info->controller_online = false;
9517 ctrl_info->pqi_mode_enabled = false;
9518
9519 return 0;
9520}
9521
9522static __maybe_unused int pqi_suspend(struct device *dev)
9523{
9524 return pqi_suspend_or_freeze(dev, true);
9525}
9526
9527static int pqi_resume_or_restore(struct device *dev)
9528{
9529 int rc;
9530 struct pci_dev *pci_dev;
9531 struct pqi_ctrl_info *ctrl_info;
9532
9533 pci_dev = to_pci_dev(dev);
9534 ctrl_info = pci_get_drvdata(pci_dev);
9535
9536 rc = pqi_request_irqs(ctrl_info);
9537 if (rc)
9538 return rc;
9539
9540 pqi_ctrl_unblock_device_reset(ctrl_info);
9541 pqi_ctrl_unblock_requests(ctrl_info);
9542 pqi_scsi_unblock_requests(ctrl_info);
9543 pqi_ctrl_unblock_scan(ctrl_info);
9544
9545 ssleep(PQI_POST_RESET_DELAY_SECS);
9546
9547 return pqi_ctrl_init_resume(ctrl_info);
9548}
9549
9550static int pqi_freeze(struct device *dev)
9551{
9552 return pqi_suspend_or_freeze(dev, false);
9553}
9554
9555static int pqi_thaw(struct device *dev)
9556{
9557 int rc;
9558 struct pci_dev *pci_dev;
9559 struct pqi_ctrl_info *ctrl_info;
9560
9561 pci_dev = to_pci_dev(dev);
9562 ctrl_info = pci_get_drvdata(pci_dev);
9563
9564 rc = pqi_request_irqs(ctrl_info);
9565 if (rc)
9566 return rc;
9567
9568 ctrl_info->controller_online = true;
9569 ctrl_info->pqi_mode_enabled = true;
9570
9571 pqi_ctrl_unblock_device_reset(ctrl_info);
9572 pqi_ctrl_unblock_requests(ctrl_info);
9573 pqi_scsi_unblock_requests(ctrl_info);
9574 pqi_ctrl_unblock_scan(ctrl_info);
9575
9576 return 0;
9577}
9578
9579static int pqi_poweroff(struct device *dev)
9580{
9581 struct pci_dev *pci_dev;
9582 struct pqi_ctrl_info *ctrl_info;
9583 enum bmic_flush_cache_shutdown_event shutdown_event;
9584
9585 pci_dev = to_pci_dev(dev);
9586 ctrl_info = pci_get_drvdata(pci_dev);
9587
9588 shutdown_event = pqi_get_flush_cache_shutdown_event(pci_dev);
9589 pqi_flush_cache(ctrl_info, shutdown_event);
9590
9591 return 0;
9592}
9593
9594static const struct dev_pm_ops pqi_pm_ops = {
9595 .suspend = pqi_suspend,
9596 .resume = pqi_resume_or_restore,
9597 .freeze = pqi_freeze,
9598 .thaw = pqi_thaw,
9599 .poweroff = pqi_poweroff,
9600 .restore = pqi_resume_or_restore,
9601};
9602
9603#endif /* CONFIG_PM */
9604
9605/* Define the PCI IDs for the controllers that we support. */
9606static const struct pci_device_id pqi_pci_id_table[] = {
9607 {
9608 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9609 0x105b, 0x1211)
9610 },
9611 {
9612 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9613 0x105b, 0x1321)
9614 },
9615 {
9616 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9617 0x152d, 0x8a22)
9618 },
9619 {
9620 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9621 0x152d, 0x8a23)
9622 },
9623 {
9624 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9625 0x152d, 0x8a24)
9626 },
9627 {
9628 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9629 0x152d, 0x8a36)
9630 },
9631 {
9632 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9633 0x152d, 0x8a37)
9634 },
9635 {
9636 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9637 0x193d, 0x0462)
9638 },
9639 {
9640 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9641 0x193d, 0x1104)
9642 },
9643 {
9644 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9645 0x193d, 0x1105)
9646 },
9647 {
9648 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9649 0x193d, 0x1106)
9650 },
9651 {
9652 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9653 0x193d, 0x1107)
9654 },
9655 {
9656 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9657 0x193d, 0x1108)
9658 },
9659 {
9660 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9661 0x193d, 0x1109)
9662 },
9663 {
9664 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9665 0x193d, 0x110b)
9666 },
9667 {
9668 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9669 0x193d, 0x1110)
9670 },
9671 {
9672 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9673 0x193d, 0x8460)
9674 },
9675 {
9676 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9677 0x193d, 0x8461)
9678 },
9679 {
9680 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9681 0x193d, 0x8462)
9682 },
9683 {
9684 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9685 0x193d, 0xc460)
9686 },
9687 {
9688 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9689 0x193d, 0xc461)
9690 },
9691 {
9692 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9693 0x193d, 0xf460)
9694 },
9695 {
9696 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9697 0x193d, 0xf461)
9698 },
9699 {
9700 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9701 0x1bd4, 0x0045)
9702 },
9703 {
9704 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9705 0x1bd4, 0x0046)
9706 },
9707 {
9708 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9709 0x1bd4, 0x0047)
9710 },
9711 {
9712 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9713 0x1bd4, 0x0048)
9714 },
9715 {
9716 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9717 0x1bd4, 0x004a)
9718 },
9719 {
9720 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9721 0x1bd4, 0x004b)
9722 },
9723 {
9724 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9725 0x1bd4, 0x004c)
9726 },
9727 {
9728 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9729 0x1bd4, 0x004f)
9730 },
9731 {
9732 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9733 0x1bd4, 0x0051)
9734 },
9735 {
9736 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9737 0x1bd4, 0x0052)
9738 },
9739 {
9740 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9741 0x1bd4, 0x0053)
9742 },
9743 {
9744 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9745 0x1bd4, 0x0054)
9746 },
9747 {
9748 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9749 0x1bd4, 0x006b)
9750 },
9751 {
9752 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9753 0x1bd4, 0x006c)
9754 },
9755 {
9756 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9757 0x1bd4, 0x006d)
9758 },
9759 {
9760 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9761 0x1bd4, 0x006f)
9762 },
9763 {
9764 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9765 0x1bd4, 0x0070)
9766 },
9767 {
9768 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9769 0x1bd4, 0x0071)
9770 },
9771 {
9772 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9773 0x1bd4, 0x0072)
9774 },
9775 {
9776 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9777 0x1bd4, 0x0086)
9778 },
9779 {
9780 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9781 0x1bd4, 0x0087)
9782 },
9783 {
9784 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9785 0x1bd4, 0x0088)
9786 },
9787 {
9788 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9789 0x1bd4, 0x0089)
9790 },
9791 {
9792 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9793 0x1bd4, 0x00a3)
9794 },
9795 {
9796 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9797 0x1ff9, 0x00a1)
9798 },
9799 {
9800 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9801 0x1f3a, 0x0104)
9802 },
9803 {
9804 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9805 0x19e5, 0xd227)
9806 },
9807 {
9808 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9809 0x19e5, 0xd228)
9810 },
9811 {
9812 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9813 0x19e5, 0xd229)
9814 },
9815 {
9816 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9817 0x19e5, 0xd22a)
9818 },
9819 {
9820 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9821 0x19e5, 0xd22b)
9822 },
9823 {
9824 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9825 0x19e5, 0xd22c)
9826 },
9827 {
9828 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9829 PCI_VENDOR_ID_ADAPTEC2, 0x0110)
9830 },
9831 {
9832 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9833 PCI_VENDOR_ID_ADAPTEC2, 0x0608)
9834 },
9835 {
9836 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9837 PCI_VENDOR_ID_ADAPTEC2, 0x0659)
9838 },
9839 {
9840 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9841 PCI_VENDOR_ID_ADAPTEC2, 0x0800)
9842 },
9843 {
9844 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9845 PCI_VENDOR_ID_ADAPTEC2, 0x0801)
9846 },
9847 {
9848 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9849 PCI_VENDOR_ID_ADAPTEC2, 0x0802)
9850 },
9851 {
9852 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9853 PCI_VENDOR_ID_ADAPTEC2, 0x0803)
9854 },
9855 {
9856 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9857 PCI_VENDOR_ID_ADAPTEC2, 0x0804)
9858 },
9859 {
9860 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9861 PCI_VENDOR_ID_ADAPTEC2, 0x0805)
9862 },
9863 {
9864 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9865 PCI_VENDOR_ID_ADAPTEC2, 0x0806)
9866 },
9867 {
9868 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9869 PCI_VENDOR_ID_ADAPTEC2, 0x0807)
9870 },
9871 {
9872 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9873 PCI_VENDOR_ID_ADAPTEC2, 0x0808)
9874 },
9875 {
9876 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9877 PCI_VENDOR_ID_ADAPTEC2, 0x0809)
9878 },
9879 {
9880 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9881 PCI_VENDOR_ID_ADAPTEC2, 0x080a)
9882 },
9883 {
9884 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9885 PCI_VENDOR_ID_ADAPTEC2, 0x0900)
9886 },
9887 {
9888 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9889 PCI_VENDOR_ID_ADAPTEC2, 0x0901)
9890 },
9891 {
9892 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9893 PCI_VENDOR_ID_ADAPTEC2, 0x0902)
9894 },
9895 {
9896 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9897 PCI_VENDOR_ID_ADAPTEC2, 0x0903)
9898 },
9899 {
9900 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9901 PCI_VENDOR_ID_ADAPTEC2, 0x0904)
9902 },
9903 {
9904 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9905 PCI_VENDOR_ID_ADAPTEC2, 0x0905)
9906 },
9907 {
9908 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9909 PCI_VENDOR_ID_ADAPTEC2, 0x0906)
9910 },
9911 {
9912 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9913 PCI_VENDOR_ID_ADAPTEC2, 0x0907)
9914 },
9915 {
9916 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9917 PCI_VENDOR_ID_ADAPTEC2, 0x0908)
9918 },
9919 {
9920 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9921 PCI_VENDOR_ID_ADAPTEC2, 0x090a)
9922 },
9923 {
9924 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9925 PCI_VENDOR_ID_ADAPTEC2, 0x1200)
9926 },
9927 {
9928 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9929 PCI_VENDOR_ID_ADAPTEC2, 0x1201)
9930 },
9931 {
9932 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9933 PCI_VENDOR_ID_ADAPTEC2, 0x1202)
9934 },
9935 {
9936 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9937 PCI_VENDOR_ID_ADAPTEC2, 0x1280)
9938 },
9939 {
9940 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9941 PCI_VENDOR_ID_ADAPTEC2, 0x1281)
9942 },
9943 {
9944 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9945 PCI_VENDOR_ID_ADAPTEC2, 0x1282)
9946 },
9947 {
9948 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9949 PCI_VENDOR_ID_ADAPTEC2, 0x1300)
9950 },
9951 {
9952 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9953 PCI_VENDOR_ID_ADAPTEC2, 0x1301)
9954 },
9955 {
9956 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9957 PCI_VENDOR_ID_ADAPTEC2, 0x1302)
9958 },
9959 {
9960 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9961 PCI_VENDOR_ID_ADAPTEC2, 0x1303)
9962 },
9963 {
9964 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9965 PCI_VENDOR_ID_ADAPTEC2, 0x1304)
9966 },
9967 {
9968 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9969 PCI_VENDOR_ID_ADAPTEC2, 0x1380)
9970 },
9971 {
9972 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9973 PCI_VENDOR_ID_ADAPTEC2, 0x1400)
9974 },
9975 {
9976 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9977 PCI_VENDOR_ID_ADAPTEC2, 0x1402)
9978 },
9979 {
9980 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9981 PCI_VENDOR_ID_ADAPTEC2, 0x1410)
9982 },
9983 {
9984 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9985 PCI_VENDOR_ID_ADAPTEC2, 0x1411)
9986 },
9987 {
9988 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9989 PCI_VENDOR_ID_ADAPTEC2, 0x1412)
9990 },
9991 {
9992 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9993 PCI_VENDOR_ID_ADAPTEC2, 0x1420)
9994 },
9995 {
9996 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
9997 PCI_VENDOR_ID_ADAPTEC2, 0x1430)
9998 },
9999 {
10000 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10001 PCI_VENDOR_ID_ADAPTEC2, 0x1440)
10002 },
10003 {
10004 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10005 PCI_VENDOR_ID_ADAPTEC2, 0x1441)
10006 },
10007 {
10008 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10009 PCI_VENDOR_ID_ADAPTEC2, 0x1450)
10010 },
10011 {
10012 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10013 PCI_VENDOR_ID_ADAPTEC2, 0x1452)
10014 },
10015 {
10016 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10017 PCI_VENDOR_ID_ADAPTEC2, 0x1460)
10018 },
10019 {
10020 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10021 PCI_VENDOR_ID_ADAPTEC2, 0x1461)
10022 },
10023 {
10024 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10025 PCI_VENDOR_ID_ADAPTEC2, 0x1462)
10026 },
10027 {
10028 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10029 PCI_VENDOR_ID_ADAPTEC2, 0x1463)
10030 },
10031 {
10032 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10033 PCI_VENDOR_ID_ADAPTEC2, 0x1470)
10034 },
10035 {
10036 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10037 PCI_VENDOR_ID_ADAPTEC2, 0x1471)
10038 },
10039 {
10040 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10041 PCI_VENDOR_ID_ADAPTEC2, 0x1472)
10042 },
10043 {
10044 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10045 PCI_VENDOR_ID_ADAPTEC2, 0x1473)
10046 },
10047 {
10048 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10049 PCI_VENDOR_ID_ADAPTEC2, 0x1474)
10050 },
10051 {
10052 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10053 PCI_VENDOR_ID_ADAPTEC2, 0x1475)
10054 },
10055 {
10056 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10057 PCI_VENDOR_ID_ADAPTEC2, 0x1480)
10058 },
10059 {
10060 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10061 PCI_VENDOR_ID_ADAPTEC2, 0x1490)
10062 },
10063 {
10064 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10065 PCI_VENDOR_ID_ADAPTEC2, 0x1491)
10066 },
10067 {
10068 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10069 PCI_VENDOR_ID_ADAPTEC2, 0x14a0)
10070 },
10071 {
10072 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10073 PCI_VENDOR_ID_ADAPTEC2, 0x14a1)
10074 },
10075 {
10076 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10077 PCI_VENDOR_ID_ADAPTEC2, 0x14a2)
10078 },
10079 {
10080 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10081 PCI_VENDOR_ID_ADAPTEC2, 0x14a4)
10082 },
10083 {
10084 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10085 PCI_VENDOR_ID_ADAPTEC2, 0x14a5)
10086 },
10087 {
10088 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10089 PCI_VENDOR_ID_ADAPTEC2, 0x14a6)
10090 },
10091 {
10092 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10093 PCI_VENDOR_ID_ADAPTEC2, 0x14b0)
10094 },
10095 {
10096 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10097 PCI_VENDOR_ID_ADAPTEC2, 0x14b1)
10098 },
10099 {
10100 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10101 PCI_VENDOR_ID_ADAPTEC2, 0x14c0)
10102 },
10103 {
10104 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10105 PCI_VENDOR_ID_ADAPTEC2, 0x14c1)
10106 },
10107 {
10108 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10109 PCI_VENDOR_ID_ADAPTEC2, 0x14c2)
10110 },
10111 {
10112 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10113 PCI_VENDOR_ID_ADAPTEC2, 0x14c3)
10114 },
10115 {
10116 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10117 PCI_VENDOR_ID_ADAPTEC2, 0x14c4)
10118 },
10119 {
10120 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10121 PCI_VENDOR_ID_ADAPTEC2, 0x14d0)
10122 },
10123 {
10124 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10125 PCI_VENDOR_ID_ADAPTEC2, 0x14e0)
10126 },
10127 {
10128 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10129 PCI_VENDOR_ID_ADAPTEC2, 0x14f0)
10130 },
10131 {
10132 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10133 0x207d, 0x4044)
10134 },
10135 {
10136 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10137 0x207d, 0x4054)
10138 },
10139 {
10140 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10141 0x207d, 0x4084)
10142 },
10143 {
10144 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10145 0x207d, 0x4094)
10146 },
10147 {
10148 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10149 0x207d, 0x4140)
10150 },
10151 {
10152 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10153 0x207d, 0x4240)
10154 },
10155 {
10156 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10157 0x207d, 0x4840)
10158 },
10159 {
10160 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10161 PCI_VENDOR_ID_ADVANTECH, 0x8312)
10162 },
10163 {
10164 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10165 PCI_VENDOR_ID_DELL, 0x1fe0)
10166 },
10167 {
10168 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10169 PCI_VENDOR_ID_HP, 0x0600)
10170 },
10171 {
10172 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10173 PCI_VENDOR_ID_HP, 0x0601)
10174 },
10175 {
10176 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10177 PCI_VENDOR_ID_HP, 0x0602)
10178 },
10179 {
10180 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10181 PCI_VENDOR_ID_HP, 0x0603)
10182 },
10183 {
10184 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10185 PCI_VENDOR_ID_HP, 0x0609)
10186 },
10187 {
10188 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10189 PCI_VENDOR_ID_HP, 0x0650)
10190 },
10191 {
10192 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10193 PCI_VENDOR_ID_HP, 0x0651)
10194 },
10195 {
10196 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10197 PCI_VENDOR_ID_HP, 0x0652)
10198 },
10199 {
10200 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10201 PCI_VENDOR_ID_HP, 0x0653)
10202 },
10203 {
10204 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10205 PCI_VENDOR_ID_HP, 0x0654)
10206 },
10207 {
10208 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10209 PCI_VENDOR_ID_HP, 0x0655)
10210 },
10211 {
10212 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10213 PCI_VENDOR_ID_HP, 0x0700)
10214 },
10215 {
10216 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10217 PCI_VENDOR_ID_HP, 0x0701)
10218 },
10219 {
10220 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10221 PCI_VENDOR_ID_HP, 0x1001)
10222 },
10223 {
10224 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10225 PCI_VENDOR_ID_HP, 0x1002)
10226 },
10227 {
10228 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10229 PCI_VENDOR_ID_HP, 0x1100)
10230 },
10231 {
10232 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10233 PCI_VENDOR_ID_HP, 0x1101)
10234 },
10235 {
10236 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10237 0x1590, 0x0294)
10238 },
10239 {
10240 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10241 0x1590, 0x02db)
10242 },
10243 {
10244 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10245 0x1590, 0x02dc)
10246 },
10247 {
10248 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10249 0x1590, 0x032e)
10250 },
10251 {
10252 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10253 0x1590, 0x036f)
10254 },
10255 {
10256 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10257 0x1590, 0x0381)
10258 },
10259 {
10260 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10261 0x1590, 0x0382)
10262 },
10263 {
10264 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10265 0x1590, 0x0383)
10266 },
10267 {
10268 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10269 0x1d8d, 0x0800)
10270 },
10271 {
10272 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10273 0x1d8d, 0x0908)
10274 },
10275 {
10276 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10277 0x1d8d, 0x0806)
10278 },
10279 {
10280 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10281 0x1d8d, 0x0916)
10282 },
10283 {
10284 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10285 PCI_VENDOR_ID_GIGABYTE, 0x1000)
10286 },
10287 {
10288 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10289 0x1dfc, 0x3161)
10290 },
10291 {
10292 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10293 0x1f0c, 0x3161)
10294 },
10295 {
10296 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10297 0x1cf2, 0x0804)
10298 },
10299 {
10300 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10301 0x1cf2, 0x0805)
10302 },
10303 {
10304 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10305 0x1cf2, 0x0806)
10306 },
10307 {
10308 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10309 0x1cf2, 0x5445)
10310 },
10311 {
10312 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10313 0x1cf2, 0x5446)
10314 },
10315 {
10316 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10317 0x1cf2, 0x5447)
10318 },
10319 {
10320 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10321 0x1cf2, 0x5449)
10322 },
10323 {
10324 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10325 0x1cf2, 0x544a)
10326 },
10327 {
10328 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10329 0x1cf2, 0x544b)
10330 },
10331 {
10332 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10333 0x1cf2, 0x544d)
10334 },
10335 {
10336 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10337 0x1cf2, 0x544e)
10338 },
10339 {
10340 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10341 0x1cf2, 0x544f)
10342 },
10343 {
10344 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10345 0x1cf2, 0x54da)
10346 },
10347 {
10348 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10349 0x1cf2, 0x54db)
10350 },
10351 {
10352 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10353 0x1cf2, 0x54dc)
10354 },
10355 {
10356 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10357 0x1cf2, 0x0b27)
10358 },
10359 {
10360 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10361 0x1cf2, 0x0b29)
10362 },
10363 {
10364 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10365 0x1cf2, 0x0b45)
10366 },
10367 {
10368 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10369 0x1cc4, 0x0101)
10370 },
10371 {
10372 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10373 0x1cc4, 0x0201)
10374 },
10375 {
10376 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10377 0x1018, 0x8238)
10378 },
10379 {
10380 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10381 0x1f3f, 0x0610)
10382 },
10383 {
10384 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10385 PCI_VENDOR_ID_LENOVO, 0x0220)
10386 },
10387 {
10388 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10389 PCI_VENDOR_ID_LENOVO, 0x0221)
10390 },
10391 {
10392 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10393 PCI_VENDOR_ID_LENOVO, 0x0222)
10394 },
10395 {
10396 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10397 PCI_VENDOR_ID_LENOVO, 0x0223)
10398 },
10399 {
10400 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10401 PCI_VENDOR_ID_LENOVO, 0x0224)
10402 },
10403 {
10404 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10405 PCI_VENDOR_ID_LENOVO, 0x0225)
10406 },
10407 {
10408 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10409 PCI_VENDOR_ID_LENOVO, 0x0520)
10410 },
10411 {
10412 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10413 PCI_VENDOR_ID_LENOVO, 0x0521)
10414 },
10415 {
10416 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10417 PCI_VENDOR_ID_LENOVO, 0x0522)
10418 },
10419 {
10420 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10421 PCI_VENDOR_ID_LENOVO, 0x0620)
10422 },
10423 {
10424 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10425 PCI_VENDOR_ID_LENOVO, 0x0621)
10426 },
10427 {
10428 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10429 PCI_VENDOR_ID_LENOVO, 0x0622)
10430 },
10431 {
10432 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10433 PCI_VENDOR_ID_LENOVO, 0x0623)
10434 },
10435 {
10436 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10437 PCI_VENDOR_ID_LENOVO, 0x0624)
10438 },
10439 {
10440 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10441 PCI_VENDOR_ID_LENOVO, 0x0625)
10442 },
10443 {
10444 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10445 PCI_VENDOR_ID_LENOVO, 0x0626)
10446 },
10447 {
10448 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10449 PCI_VENDOR_ID_LENOVO, 0x0627)
10450 },
10451 {
10452 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10453 PCI_VENDOR_ID_LENOVO, 0x0628)
10454 },
10455 {
10456 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10457 0x1014, 0x0718)
10458 },
10459 {
10460 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10461 0x1137, 0x02f8)
10462 },
10463 {
10464 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10465 0x1137, 0x02f9)
10466 },
10467 {
10468 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10469 0x1137, 0x02fa)
10470 },
10471 {
10472 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10473 0x1137, 0x02fe)
10474 },
10475 {
10476 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10477 0x1137, 0x02ff)
10478 },
10479 {
10480 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10481 0x1137, 0x0300)
10482 },
10483 {
10484 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10485 0x1ded, 0x3301)
10486 },
10487 {
10488 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10489 0x1ff9, 0x0045)
10490 },
10491 {
10492 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10493 0x1ff9, 0x0046)
10494 },
10495 {
10496 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10497 0x1ff9, 0x0047)
10498 },
10499 {
10500 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10501 0x1ff9, 0x0048)
10502 },
10503 {
10504 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10505 0x1ff9, 0x004a)
10506 },
10507 {
10508 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10509 0x1ff9, 0x004b)
10510 },
10511 {
10512 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10513 0x1ff9, 0x004c)
10514 },
10515 {
10516 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10517 0x1ff9, 0x004f)
10518 },
10519 {
10520 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10521 0x1ff9, 0x0051)
10522 },
10523 {
10524 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10525 0x1ff9, 0x0052)
10526 },
10527 {
10528 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10529 0x1ff9, 0x0053)
10530 },
10531 {
10532 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10533 0x1ff9, 0x0054)
10534 },
10535 {
10536 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10537 0x1ff9, 0x006b)
10538 },
10539 {
10540 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10541 0x1ff9, 0x006c)
10542 },
10543 {
10544 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10545 0x1ff9, 0x006d)
10546 },
10547 {
10548 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10549 0x1ff9, 0x006f)
10550 },
10551 {
10552 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10553 0x1ff9, 0x0070)
10554 },
10555 {
10556 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10557 0x1ff9, 0x0071)
10558 },
10559 {
10560 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10561 0x1ff9, 0x0072)
10562 },
10563 {
10564 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10565 0x1ff9, 0x0086)
10566 },
10567 {
10568 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10569 0x1ff9, 0x0087)
10570 },
10571 {
10572 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10573 0x1ff9, 0x0088)
10574 },
10575 {
10576 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10577 0x1ff9, 0x0089)
10578 },
10579 {
10580 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10581 0x1e93, 0x1000)
10582 },
10583 {
10584 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10585 0x1e93, 0x1001)
10586 },
10587 {
10588 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10589 0x1e93, 0x1002)
10590 },
10591 {
10592 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10593 0x1e93, 0x1005)
10594 },
10595 {
10596 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10597 0x1f51, 0x1001)
10598 },
10599 {
10600 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10601 0x1f51, 0x1002)
10602 },
10603 {
10604 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10605 0x1f51, 0x1003)
10606 },
10607 {
10608 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10609 0x1f51, 0x1004)
10610 },
10611 {
10612 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10613 0x1f51, 0x1005)
10614 },
10615 {
10616 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10617 0x1f51, 0x1006)
10618 },
10619 {
10620 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10621 0x1f51, 0x1007)
10622 },
10623 {
10624 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10625 0x1f51, 0x1008)
10626 },
10627 {
10628 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10629 0x1f51, 0x1009)
10630 },
10631 {
10632 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10633 0x1f51, 0x100a)
10634 },
10635 {
10636 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10637 0x1f51, 0x100b)
10638 },
10639 {
10640 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10641 0x1f51, 0x100e)
10642 },
10643 {
10644 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10645 0x1f51, 0x100f)
10646 },
10647 {
10648 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10649 0x1f51, 0x1010)
10650 },
10651 {
10652 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10653 0x1f51, 0x1011)
10654 },
10655 {
10656 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10657 0x1f51, 0x1043)
10658 },
10659 {
10660 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10661 0x1f51, 0x1044)
10662 },
10663 {
10664 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10665 0x1f51, 0x1045)
10666 },
10667 {
10668 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10669 0x1ff9, 0x00a3)
10670 },
10671 {
10672 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
10673 PCI_ANY_ID, PCI_ANY_ID)
10674 },
10675 { 0 }
10676};
10677
10678MODULE_DEVICE_TABLE(pci, pqi_pci_id_table);
10679
10680static struct pci_driver pqi_pci_driver = {
10681 .name = DRIVER_NAME_SHORT,
10682 .id_table = pqi_pci_id_table,
10683 .probe = pqi_pci_probe,
10684 .remove = pqi_pci_remove,
10685 .shutdown = pqi_shutdown,
10686#if defined(CONFIG_PM)
10687 .driver = {
10688 .pm = &pqi_pm_ops
10689 },
10690#endif
10691};
10692
10693static int __init pqi_init(void)
10694{
10695 int rc;
10696
10697 pr_info(DRIVER_NAME "\n");
10698 pqi_verify_structures();
10699 sis_verify_structures();
10700
10701 pqi_sas_transport_template = sas_attach_transport(&pqi_sas_transport_functions);
10702 if (!pqi_sas_transport_template)
10703 return -ENODEV;
10704
10705 pqi_process_module_params();
10706
10707 rc = pci_register_driver(&pqi_pci_driver);
10708 if (rc)
10709 sas_release_transport(pqi_sas_transport_template);
10710
10711 return rc;
10712}
10713
10714static void __exit pqi_cleanup(void)
10715{
10716 pci_unregister_driver(&pqi_pci_driver);
10717 sas_release_transport(pqi_sas_transport_template);
10718}
10719
10720module_init(pqi_init);
10721module_exit(pqi_cleanup);
10722
10723static void pqi_verify_structures(void)
10724{
10725 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
10726 sis_host_to_ctrl_doorbell) != 0x20);
10727 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
10728 sis_interrupt_mask) != 0x34);
10729 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
10730 sis_ctrl_to_host_doorbell) != 0x9c);
10731 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
10732 sis_ctrl_to_host_doorbell_clear) != 0xa0);
10733 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
10734 sis_driver_scratch) != 0xb0);
10735 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
10736 sis_product_identifier) != 0xb4);
10737 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
10738 sis_firmware_status) != 0xbc);
10739 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
10740 sis_ctrl_shutdown_reason_code) != 0xcc);
10741 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
10742 sis_mailbox) != 0x1000);
10743 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
10744 pqi_registers) != 0x4000);
10745
10746 BUILD_BUG_ON(offsetof(struct pqi_iu_header,
10747 iu_type) != 0x0);
10748 BUILD_BUG_ON(offsetof(struct pqi_iu_header,
10749 iu_length) != 0x2);
10750 BUILD_BUG_ON(offsetof(struct pqi_iu_header,
10751 response_queue_id) != 0x4);
10752 BUILD_BUG_ON(offsetof(struct pqi_iu_header,
10753 driver_flags) != 0x6);
10754 BUILD_BUG_ON(sizeof(struct pqi_iu_header) != 0x8);
10755
10756 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
10757 status) != 0x0);
10758 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
10759 service_response) != 0x1);
10760 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
10761 data_present) != 0x2);
10762 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
10763 reserved) != 0x3);
10764 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
10765 residual_count) != 0x4);
10766 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
10767 data_length) != 0x8);
10768 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
10769 reserved1) != 0xa);
10770 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
10771 data) != 0xc);
10772 BUILD_BUG_ON(sizeof(struct pqi_aio_error_info) != 0x10c);
10773
10774 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
10775 data_in_result) != 0x0);
10776 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
10777 data_out_result) != 0x1);
10778 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
10779 reserved) != 0x2);
10780 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
10781 status) != 0x5);
10782 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
10783 status_qualifier) != 0x6);
10784 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
10785 sense_data_length) != 0x8);
10786 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
10787 response_data_length) != 0xa);
10788 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
10789 data_in_transferred) != 0xc);
10790 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
10791 data_out_transferred) != 0x10);
10792 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
10793 data) != 0x14);
10794 BUILD_BUG_ON(sizeof(struct pqi_raid_error_info) != 0x114);
10795
10796 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10797 signature) != 0x0);
10798 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10799 function_and_status_code) != 0x8);
10800 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10801 max_admin_iq_elements) != 0x10);
10802 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10803 max_admin_oq_elements) != 0x11);
10804 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10805 admin_iq_element_length) != 0x12);
10806 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10807 admin_oq_element_length) != 0x13);
10808 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10809 max_reset_timeout) != 0x14);
10810 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10811 legacy_intx_status) != 0x18);
10812 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10813 legacy_intx_mask_set) != 0x1c);
10814 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10815 legacy_intx_mask_clear) != 0x20);
10816 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10817 device_status) != 0x40);
10818 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10819 admin_iq_pi_offset) != 0x48);
10820 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10821 admin_oq_ci_offset) != 0x50);
10822 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10823 admin_iq_element_array_addr) != 0x58);
10824 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10825 admin_oq_element_array_addr) != 0x60);
10826 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10827 admin_iq_ci_addr) != 0x68);
10828 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10829 admin_oq_pi_addr) != 0x70);
10830 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10831 admin_iq_num_elements) != 0x78);
10832 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10833 admin_oq_num_elements) != 0x79);
10834 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10835 admin_queue_int_msg_num) != 0x7a);
10836 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10837 device_error) != 0x80);
10838 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10839 error_details) != 0x88);
10840 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10841 device_reset) != 0x90);
10842 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
10843 power_action) != 0x94);
10844 BUILD_BUG_ON(sizeof(struct pqi_device_registers) != 0x100);
10845
10846 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10847 header.iu_type) != 0);
10848 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10849 header.iu_length) != 2);
10850 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10851 header.driver_flags) != 6);
10852 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10853 request_id) != 8);
10854 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10855 function_code) != 10);
10856 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10857 data.report_device_capability.buffer_length) != 44);
10858 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10859 data.report_device_capability.sg_descriptor) != 48);
10860 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10861 data.create_operational_iq.queue_id) != 12);
10862 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10863 data.create_operational_iq.element_array_addr) != 16);
10864 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10865 data.create_operational_iq.ci_addr) != 24);
10866 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10867 data.create_operational_iq.num_elements) != 32);
10868 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10869 data.create_operational_iq.element_length) != 34);
10870 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10871 data.create_operational_iq.queue_protocol) != 36);
10872 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10873 data.create_operational_oq.queue_id) != 12);
10874 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10875 data.create_operational_oq.element_array_addr) != 16);
10876 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10877 data.create_operational_oq.pi_addr) != 24);
10878 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10879 data.create_operational_oq.num_elements) != 32);
10880 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10881 data.create_operational_oq.element_length) != 34);
10882 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10883 data.create_operational_oq.queue_protocol) != 36);
10884 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10885 data.create_operational_oq.int_msg_num) != 40);
10886 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10887 data.create_operational_oq.coalescing_count) != 42);
10888 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10889 data.create_operational_oq.min_coalescing_time) != 44);
10890 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10891 data.create_operational_oq.max_coalescing_time) != 48);
10892 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
10893 data.delete_operational_queue.queue_id) != 12);
10894 BUILD_BUG_ON(sizeof(struct pqi_general_admin_request) != 64);
10895 BUILD_BUG_ON(sizeof_field(struct pqi_general_admin_request,
10896 data.create_operational_iq) != 64 - 11);
10897 BUILD_BUG_ON(sizeof_field(struct pqi_general_admin_request,
10898 data.create_operational_oq) != 64 - 11);
10899 BUILD_BUG_ON(sizeof_field(struct pqi_general_admin_request,
10900 data.delete_operational_queue) != 64 - 11);
10901
10902 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
10903 header.iu_type) != 0);
10904 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
10905 header.iu_length) != 2);
10906 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
10907 header.driver_flags) != 6);
10908 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
10909 request_id) != 8);
10910 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
10911 function_code) != 10);
10912 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
10913 status) != 11);
10914 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
10915 data.create_operational_iq.status_descriptor) != 12);
10916 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
10917 data.create_operational_iq.iq_pi_offset) != 16);
10918 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
10919 data.create_operational_oq.status_descriptor) != 12);
10920 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
10921 data.create_operational_oq.oq_ci_offset) != 16);
10922 BUILD_BUG_ON(sizeof(struct pqi_general_admin_response) != 64);
10923
10924 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10925 header.iu_type) != 0);
10926 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10927 header.iu_length) != 2);
10928 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10929 header.response_queue_id) != 4);
10930 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10931 header.driver_flags) != 6);
10932 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10933 request_id) != 8);
10934 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10935 nexus_id) != 10);
10936 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10937 buffer_length) != 12);
10938 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10939 lun_number) != 16);
10940 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10941 protocol_specific) != 24);
10942 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10943 error_index) != 27);
10944 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10945 cdb) != 32);
10946 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10947 timeout) != 60);
10948 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
10949 sg_descriptors) != 64);
10950 BUILD_BUG_ON(sizeof(struct pqi_raid_path_request) !=
10951 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
10952
10953 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10954 header.iu_type) != 0);
10955 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10956 header.iu_length) != 2);
10957 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10958 header.response_queue_id) != 4);
10959 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10960 header.driver_flags) != 6);
10961 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10962 request_id) != 8);
10963 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10964 nexus_id) != 12);
10965 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10966 buffer_length) != 16);
10967 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10968 data_encryption_key_index) != 22);
10969 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10970 encrypt_tweak_lower) != 24);
10971 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10972 encrypt_tweak_upper) != 28);
10973 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10974 cdb) != 32);
10975 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10976 error_index) != 48);
10977 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10978 num_sg_descriptors) != 50);
10979 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10980 cdb_length) != 51);
10981 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10982 lun_number) != 52);
10983 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
10984 sg_descriptors) != 64);
10985 BUILD_BUG_ON(sizeof(struct pqi_aio_path_request) !=
10986 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
10987
10988 BUILD_BUG_ON(offsetof(struct pqi_io_response,
10989 header.iu_type) != 0);
10990 BUILD_BUG_ON(offsetof(struct pqi_io_response,
10991 header.iu_length) != 2);
10992 BUILD_BUG_ON(offsetof(struct pqi_io_response,
10993 request_id) != 8);
10994 BUILD_BUG_ON(offsetof(struct pqi_io_response,
10995 error_index) != 10);
10996
10997 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
10998 header.iu_type) != 0);
10999 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
11000 header.iu_length) != 2);
11001 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
11002 header.response_queue_id) != 4);
11003 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
11004 request_id) != 8);
11005 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
11006 data.report_event_configuration.buffer_length) != 12);
11007 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
11008 data.report_event_configuration.sg_descriptors) != 16);
11009 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
11010 data.set_event_configuration.global_event_oq_id) != 10);
11011 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
11012 data.set_event_configuration.buffer_length) != 12);
11013 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
11014 data.set_event_configuration.sg_descriptors) != 16);
11015
11016 BUILD_BUG_ON(offsetof(struct pqi_iu_layer_descriptor,
11017 max_inbound_iu_length) != 6);
11018 BUILD_BUG_ON(offsetof(struct pqi_iu_layer_descriptor,
11019 max_outbound_iu_length) != 14);
11020 BUILD_BUG_ON(sizeof(struct pqi_iu_layer_descriptor) != 16);
11021
11022 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
11023 data_length) != 0);
11024 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
11025 iq_arbitration_priority_support_bitmask) != 8);
11026 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
11027 maximum_aw_a) != 9);
11028 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
11029 maximum_aw_b) != 10);
11030 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
11031 maximum_aw_c) != 11);
11032 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
11033 max_inbound_queues) != 16);
11034 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
11035 max_elements_per_iq) != 18);
11036 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
11037 max_iq_element_length) != 24);
11038 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
11039 min_iq_element_length) != 26);
11040 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
11041 max_outbound_queues) != 30);
11042 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
11043 max_elements_per_oq) != 32);
11044 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
11045 intr_coalescing_time_granularity) != 34);
11046 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
11047 max_oq_element_length) != 36);
11048 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
11049 min_oq_element_length) != 38);
11050 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
11051 iu_layer_descriptors) != 64);
11052 BUILD_BUG_ON(sizeof(struct pqi_device_capability) != 576);
11053
11054 BUILD_BUG_ON(offsetof(struct pqi_event_descriptor,
11055 event_type) != 0);
11056 BUILD_BUG_ON(offsetof(struct pqi_event_descriptor,
11057 oq_id) != 2);
11058 BUILD_BUG_ON(sizeof(struct pqi_event_descriptor) != 4);
11059
11060 BUILD_BUG_ON(offsetof(struct pqi_event_config,
11061 num_event_descriptors) != 2);
11062 BUILD_BUG_ON(offsetof(struct pqi_event_config,
11063 descriptors) != 4);
11064
11065 BUILD_BUG_ON(PQI_NUM_SUPPORTED_EVENTS !=
11066 ARRAY_SIZE(pqi_supported_event_types));
11067
11068 BUILD_BUG_ON(offsetof(struct pqi_event_response,
11069 header.iu_type) != 0);
11070 BUILD_BUG_ON(offsetof(struct pqi_event_response,
11071 header.iu_length) != 2);
11072 BUILD_BUG_ON(offsetof(struct pqi_event_response,
11073 event_type) != 8);
11074 BUILD_BUG_ON(offsetof(struct pqi_event_response,
11075 event_id) != 10);
11076 BUILD_BUG_ON(offsetof(struct pqi_event_response,
11077 additional_event_id) != 12);
11078 BUILD_BUG_ON(offsetof(struct pqi_event_response,
11079 data) != 16);
11080 BUILD_BUG_ON(sizeof(struct pqi_event_response) != 32);
11081
11082 BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
11083 header.iu_type) != 0);
11084 BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
11085 header.iu_length) != 2);
11086 BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
11087 event_type) != 8);
11088 BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
11089 event_id) != 10);
11090 BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
11091 additional_event_id) != 12);
11092 BUILD_BUG_ON(sizeof(struct pqi_event_acknowledge_request) != 16);
11093
11094 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
11095 header.iu_type) != 0);
11096 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
11097 header.iu_length) != 2);
11098 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
11099 request_id) != 8);
11100 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
11101 nexus_id) != 10);
11102 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
11103 timeout) != 14);
11104 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
11105 lun_number) != 16);
11106 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
11107 protocol_specific) != 24);
11108 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
11109 outbound_queue_id_to_manage) != 26);
11110 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
11111 request_id_to_manage) != 28);
11112 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
11113 task_management_function) != 30);
11114 BUILD_BUG_ON(sizeof(struct pqi_task_management_request) != 32);
11115
11116 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
11117 header.iu_type) != 0);
11118 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
11119 header.iu_length) != 2);
11120 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
11121 request_id) != 8);
11122 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
11123 nexus_id) != 10);
11124 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
11125 additional_response_info) != 12);
11126 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
11127 response_code) != 15);
11128 BUILD_BUG_ON(sizeof(struct pqi_task_management_response) != 16);
11129
11130 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
11131 configured_logical_drive_count) != 0);
11132 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
11133 configuration_signature) != 1);
11134 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
11135 firmware_version_short) != 5);
11136 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
11137 extended_logical_unit_count) != 154);
11138 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
11139 firmware_build_number) != 190);
11140 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
11141 vendor_id) != 200);
11142 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
11143 product_id) != 208);
11144 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
11145 extra_controller_flags) != 286);
11146 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
11147 controller_mode) != 292);
11148 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
11149 spare_part_number) != 293);
11150 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
11151 firmware_version_long) != 325);
11152
11153 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
11154 phys_bay_in_box) != 115);
11155 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
11156 device_type) != 120);
11157 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
11158 redundant_path_present_map) != 1736);
11159 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
11160 active_path_number) != 1738);
11161 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
11162 alternate_paths_phys_connector) != 1739);
11163 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
11164 alternate_paths_phys_box_on_port) != 1755);
11165 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
11166 current_queue_depth_limit) != 1796);
11167 BUILD_BUG_ON(sizeof(struct bmic_identify_physical_device) != 2560);
11168
11169 BUILD_BUG_ON(sizeof(struct bmic_sense_feature_buffer_header) != 4);
11170 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_buffer_header,
11171 page_code) != 0);
11172 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_buffer_header,
11173 subpage_code) != 1);
11174 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_buffer_header,
11175 buffer_length) != 2);
11176
11177 BUILD_BUG_ON(sizeof(struct bmic_sense_feature_page_header) != 4);
11178 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_page_header,
11179 page_code) != 0);
11180 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_page_header,
11181 subpage_code) != 1);
11182 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_page_header,
11183 page_length) != 2);
11184
11185 BUILD_BUG_ON(sizeof(struct bmic_sense_feature_io_page_aio_subpage)
11186 != 18);
11187 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
11188 header) != 0);
11189 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
11190 firmware_read_support) != 4);
11191 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
11192 driver_read_support) != 5);
11193 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
11194 firmware_write_support) != 6);
11195 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
11196 driver_write_support) != 7);
11197 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
11198 max_transfer_encrypted_sas_sata) != 8);
11199 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
11200 max_transfer_encrypted_nvme) != 10);
11201 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
11202 max_write_raid_5_6) != 12);
11203 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
11204 max_write_raid_1_10_2drive) != 14);
11205 BUILD_BUG_ON(offsetof(struct bmic_sense_feature_io_page_aio_subpage,
11206 max_write_raid_1_10_3drive) != 16);
11207
11208 BUILD_BUG_ON(PQI_ADMIN_IQ_NUM_ELEMENTS > 255);
11209 BUILD_BUG_ON(PQI_ADMIN_OQ_NUM_ELEMENTS > 255);
11210 BUILD_BUG_ON(PQI_ADMIN_IQ_ELEMENT_LENGTH %
11211 PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
11212 BUILD_BUG_ON(PQI_ADMIN_OQ_ELEMENT_LENGTH %
11213 PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
11214 BUILD_BUG_ON(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH > 1048560);
11215 BUILD_BUG_ON(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH %
11216 PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
11217 BUILD_BUG_ON(PQI_OPERATIONAL_OQ_ELEMENT_LENGTH > 1048560);
11218 BUILD_BUG_ON(PQI_OPERATIONAL_OQ_ELEMENT_LENGTH %
11219 PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
11220
11221 BUILD_BUG_ON(PQI_RESERVED_IO_SLOTS >= PQI_MAX_OUTSTANDING_REQUESTS);
11222 BUILD_BUG_ON(PQI_RESERVED_IO_SLOTS >=
11223 PQI_MAX_OUTSTANDING_REQUESTS_KDUMP);
11224}