2 * sata_nv.c - NVIDIA nForce SATA
4 * Copyright 2004 NVIDIA Corp. All rights reserved.
5 * Copyright 2004 Andrew Chew
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program; see the file COPYING. If not, write to
20 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
23 * libata documentation is available via 'make {ps|pdf}docs',
24 * as Documentation/DocBook/libata.*
26 * No hardware documentation available outside of NVIDIA.
27 * This driver programs the NVIDIA SATA controller in a similar
28 * fashion as with other PCI IDE BMDMA controllers, with a few
29 * NV-specific details such as register offsets, SATA phy location,
32 * CK804/MCP04 controllers support an alternate programming interface
33 * similar to the ADMA specification (with some modifications).
34 * This allows the use of NCQ. Non-DMA-mapped ATA commands are still
35 * sent through the legacy interface.
39 #include <linux/kernel.h>
40 #include <linux/module.h>
41 #include <linux/pci.h>
42 #include <linux/init.h>
43 #include <linux/blkdev.h>
44 #include <linux/delay.h>
45 #include <linux/interrupt.h>
46 #include <linux/device.h>
47 #include <scsi/scsi_host.h>
48 #include <scsi/scsi_device.h>
49 #include <linux/libata.h>
51 #define DRV_NAME "sata_nv"
52 #define DRV_VERSION "3.3"
54 #define NV_ADMA_DMA_BOUNDARY 0xffffffffUL
63 NV_PORT0_SCR_REG_OFFSET = 0x00,
64 NV_PORT1_SCR_REG_OFFSET = 0x40,
66 /* INT_STATUS/ENABLE */
69 NV_INT_STATUS_CK804 = 0x440,
70 NV_INT_ENABLE_CK804 = 0x441,
72 /* INT_STATUS/ENABLE bits */
76 NV_INT_REMOVED = 0x08,
78 NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */
81 NV_INT_MASK = NV_INT_DEV |
82 NV_INT_ADDED | NV_INT_REMOVED,
86 NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI
88 // For PCI config register 20
89 NV_MCP_SATA_CFG_20 = 0x50,
90 NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
91 NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17),
92 NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16),
93 NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14),
94 NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12),
96 NV_ADMA_MAX_CPBS = 32,
99 NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) /
101 NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5,
102 NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
103 NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS *
104 (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
106 /* BAR5 offset to ADMA general registers */
108 NV_ADMA_GEN_CTL = 0x00,
109 NV_ADMA_NOTIFIER_CLEAR = 0x30,
111 /* BAR5 offset to ADMA ports */
112 NV_ADMA_PORT = 0x480,
114 /* size of ADMA port register space */
115 NV_ADMA_PORT_SIZE = 0x100,
117 /* ADMA port registers */
119 NV_ADMA_CPB_COUNT = 0x42,
120 NV_ADMA_NEXT_CPB_IDX = 0x43,
122 NV_ADMA_CPB_BASE_LOW = 0x48,
123 NV_ADMA_CPB_BASE_HIGH = 0x4C,
124 NV_ADMA_APPEND = 0x50,
125 NV_ADMA_NOTIFIER = 0x68,
126 NV_ADMA_NOTIFIER_ERROR = 0x6C,
128 /* NV_ADMA_CTL register bits */
129 NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0),
130 NV_ADMA_CTL_CHANNEL_RESET = (1 << 5),
131 NV_ADMA_CTL_GO = (1 << 7),
132 NV_ADMA_CTL_AIEN = (1 << 8),
133 NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11),
134 NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12),
136 /* CPB response flag bits */
137 NV_CPB_RESP_DONE = (1 << 0),
138 NV_CPB_RESP_ATA_ERR = (1 << 3),
139 NV_CPB_RESP_CMD_ERR = (1 << 4),
140 NV_CPB_RESP_CPB_ERR = (1 << 7),
142 /* CPB control flag bits */
143 NV_CPB_CTL_CPB_VALID = (1 << 0),
144 NV_CPB_CTL_QUEUE = (1 << 1),
145 NV_CPB_CTL_APRD_VALID = (1 << 2),
146 NV_CPB_CTL_IEN = (1 << 3),
147 NV_CPB_CTL_FPDMA = (1 << 4),
150 NV_APRD_WRITE = (1 << 1),
151 NV_APRD_END = (1 << 2),
152 NV_APRD_CONT = (1 << 3),
154 /* NV_ADMA_STAT flags */
155 NV_ADMA_STAT_TIMEOUT = (1 << 0),
156 NV_ADMA_STAT_HOTUNPLUG = (1 << 1),
157 NV_ADMA_STAT_HOTPLUG = (1 << 2),
158 NV_ADMA_STAT_CPBERR = (1 << 4),
159 NV_ADMA_STAT_SERROR = (1 << 5),
160 NV_ADMA_STAT_CMD_COMPLETE = (1 << 6),
161 NV_ADMA_STAT_IDLE = (1 << 8),
162 NV_ADMA_STAT_LEGACY = (1 << 9),
163 NV_ADMA_STAT_STOPPED = (1 << 10),
164 NV_ADMA_STAT_DONE = (1 << 12),
165 NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR |
166 NV_ADMA_STAT_TIMEOUT,
169 NV_ADMA_PORT_REGISTER_MODE = (1 << 0),
170 NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1),
174 /* ADMA Physical Region Descriptor - one SG segment */
183 enum nv_adma_regbits {
184 CMDEND = (1 << 15), /* end of command list */
185 WNB = (1 << 14), /* wait-not-BSY */
186 IGN = (1 << 13), /* ignore this entry */
187 CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */
188 DA2 = (1 << (2 + 8)),
189 DA1 = (1 << (1 + 8)),
190 DA0 = (1 << (0 + 8)),
193 /* ADMA Command Parameter Block
194 The first 5 SG segments are stored inside the Command Parameter Block itself.
195 If there are more than 5 segments the remainder are stored in a separate
196 memory area indicated by next_aprd. */
198 u8 resp_flags; /* 0 */
199 u8 reserved1; /* 1 */
200 u8 ctl_flags; /* 2 */
201 /* len is length of taskfile in 64 bit words */
204 u8 next_cpb_idx; /* 5 */
205 __le16 reserved2; /* 6-7 */
206 __le16 tf[12]; /* 8-31 */
207 struct nv_adma_prd aprd[5]; /* 32-111 */
208 __le64 next_aprd; /* 112-119 */
209 __le64 reserved3; /* 120-127 */
213 struct nv_adma_port_priv {
214 struct nv_adma_cpb *cpb;
216 struct nv_adma_prd *aprd;
218 void __iomem * ctl_block;
219 void __iomem * gen_block;
220 void __iomem * notifier_clear_block;
224 struct nv_host_priv {
228 #define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & ( 1 << (19 + (12 * (PORT)))))
230 static int nv_init_one (struct pci_dev *pdev, const struct pci_device_id *ent);
231 static void nv_remove_one (struct pci_dev *pdev);
232 static int nv_pci_device_resume(struct pci_dev *pdev);
233 static void nv_ck804_host_stop(struct ata_host *host);
234 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
235 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
236 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
237 static u32 nv_scr_read (struct ata_port *ap, unsigned int sc_reg);
238 static void nv_scr_write (struct ata_port *ap, unsigned int sc_reg, u32 val);
240 static void nv_nf2_freeze(struct ata_port *ap);
241 static void nv_nf2_thaw(struct ata_port *ap);
242 static void nv_ck804_freeze(struct ata_port *ap);
243 static void nv_ck804_thaw(struct ata_port *ap);
244 static void nv_error_handler(struct ata_port *ap);
245 static int nv_adma_slave_config(struct scsi_device *sdev);
246 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
247 static void nv_adma_qc_prep(struct ata_queued_cmd *qc);
248 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
249 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
250 static void nv_adma_irq_clear(struct ata_port *ap);
251 static int nv_adma_port_start(struct ata_port *ap);
252 static void nv_adma_port_stop(struct ata_port *ap);
253 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
254 static int nv_adma_port_resume(struct ata_port *ap);
255 static void nv_adma_error_handler(struct ata_port *ap);
256 static void nv_adma_host_stop(struct ata_host *host);
257 static void nv_adma_bmdma_setup(struct ata_queued_cmd *qc);
258 static void nv_adma_bmdma_start(struct ata_queued_cmd *qc);
259 static void nv_adma_bmdma_stop(struct ata_queued_cmd *qc);
260 static u8 nv_adma_bmdma_status(struct ata_port *ap);
266 NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */
271 static const struct pci_device_id nv_pci_tbl[] = {
272 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 },
273 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 },
274 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 },
275 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 },
276 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 },
277 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 },
278 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 },
279 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), GENERIC },
280 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), GENERIC },
281 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), GENERIC },
282 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), GENERIC },
283 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC },
284 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC },
285 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC },
286 { PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID,
287 PCI_ANY_ID, PCI_ANY_ID,
288 PCI_CLASS_STORAGE_IDE<<8, 0xffff00, GENERIC },
289 { PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID,
290 PCI_ANY_ID, PCI_ANY_ID,
291 PCI_CLASS_STORAGE_RAID<<8, 0xffff00, GENERIC },
293 { } /* terminate list */
296 static struct pci_driver nv_pci_driver = {
298 .id_table = nv_pci_tbl,
299 .probe = nv_init_one,
300 .suspend = ata_pci_device_suspend,
301 .resume = nv_pci_device_resume,
302 .remove = nv_remove_one,
305 static struct scsi_host_template nv_sht = {
306 .module = THIS_MODULE,
308 .ioctl = ata_scsi_ioctl,
309 .queuecommand = ata_scsi_queuecmd,
310 .can_queue = ATA_DEF_QUEUE,
311 .this_id = ATA_SHT_THIS_ID,
312 .sg_tablesize = LIBATA_MAX_PRD,
313 .cmd_per_lun = ATA_SHT_CMD_PER_LUN,
314 .emulated = ATA_SHT_EMULATED,
315 .use_clustering = ATA_SHT_USE_CLUSTERING,
316 .proc_name = DRV_NAME,
317 .dma_boundary = ATA_DMA_BOUNDARY,
318 .slave_configure = ata_scsi_slave_config,
319 .slave_destroy = ata_scsi_slave_destroy,
320 .bios_param = ata_std_bios_param,
321 .suspend = ata_scsi_device_suspend,
322 .resume = ata_scsi_device_resume,
325 static struct scsi_host_template nv_adma_sht = {
326 .module = THIS_MODULE,
328 .ioctl = ata_scsi_ioctl,
329 .queuecommand = ata_scsi_queuecmd,
330 .can_queue = NV_ADMA_MAX_CPBS,
331 .this_id = ATA_SHT_THIS_ID,
332 .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN,
333 .cmd_per_lun = ATA_SHT_CMD_PER_LUN,
334 .emulated = ATA_SHT_EMULATED,
335 .use_clustering = ATA_SHT_USE_CLUSTERING,
336 .proc_name = DRV_NAME,
337 .dma_boundary = NV_ADMA_DMA_BOUNDARY,
338 .slave_configure = nv_adma_slave_config,
339 .slave_destroy = ata_scsi_slave_destroy,
340 .bios_param = ata_std_bios_param,
341 .suspend = ata_scsi_device_suspend,
342 .resume = ata_scsi_device_resume,
345 static const struct ata_port_operations nv_generic_ops = {
346 .port_disable = ata_port_disable,
347 .tf_load = ata_tf_load,
348 .tf_read = ata_tf_read,
349 .exec_command = ata_exec_command,
350 .check_status = ata_check_status,
351 .dev_select = ata_std_dev_select,
352 .bmdma_setup = ata_bmdma_setup,
353 .bmdma_start = ata_bmdma_start,
354 .bmdma_stop = ata_bmdma_stop,
355 .bmdma_status = ata_bmdma_status,
356 .qc_prep = ata_qc_prep,
357 .qc_issue = ata_qc_issue_prot,
358 .freeze = ata_bmdma_freeze,
359 .thaw = ata_bmdma_thaw,
360 .error_handler = nv_error_handler,
361 .post_internal_cmd = ata_bmdma_post_internal_cmd,
362 .data_xfer = ata_data_xfer,
363 .irq_handler = nv_generic_interrupt,
364 .irq_clear = ata_bmdma_irq_clear,
365 .irq_on = ata_irq_on,
366 .irq_ack = ata_irq_ack,
367 .scr_read = nv_scr_read,
368 .scr_write = nv_scr_write,
369 .port_start = ata_port_start,
372 static const struct ata_port_operations nv_nf2_ops = {
373 .port_disable = ata_port_disable,
374 .tf_load = ata_tf_load,
375 .tf_read = ata_tf_read,
376 .exec_command = ata_exec_command,
377 .check_status = ata_check_status,
378 .dev_select = ata_std_dev_select,
379 .bmdma_setup = ata_bmdma_setup,
380 .bmdma_start = ata_bmdma_start,
381 .bmdma_stop = ata_bmdma_stop,
382 .bmdma_status = ata_bmdma_status,
383 .qc_prep = ata_qc_prep,
384 .qc_issue = ata_qc_issue_prot,
385 .freeze = nv_nf2_freeze,
387 .error_handler = nv_error_handler,
388 .post_internal_cmd = ata_bmdma_post_internal_cmd,
389 .data_xfer = ata_data_xfer,
390 .irq_handler = nv_nf2_interrupt,
391 .irq_clear = ata_bmdma_irq_clear,
392 .irq_on = ata_irq_on,
393 .irq_ack = ata_irq_ack,
394 .scr_read = nv_scr_read,
395 .scr_write = nv_scr_write,
396 .port_start = ata_port_start,
399 static const struct ata_port_operations nv_ck804_ops = {
400 .port_disable = ata_port_disable,
401 .tf_load = ata_tf_load,
402 .tf_read = ata_tf_read,
403 .exec_command = ata_exec_command,
404 .check_status = ata_check_status,
405 .dev_select = ata_std_dev_select,
406 .bmdma_setup = ata_bmdma_setup,
407 .bmdma_start = ata_bmdma_start,
408 .bmdma_stop = ata_bmdma_stop,
409 .bmdma_status = ata_bmdma_status,
410 .qc_prep = ata_qc_prep,
411 .qc_issue = ata_qc_issue_prot,
412 .freeze = nv_ck804_freeze,
413 .thaw = nv_ck804_thaw,
414 .error_handler = nv_error_handler,
415 .post_internal_cmd = ata_bmdma_post_internal_cmd,
416 .data_xfer = ata_data_xfer,
417 .irq_handler = nv_ck804_interrupt,
418 .irq_clear = ata_bmdma_irq_clear,
419 .irq_on = ata_irq_on,
420 .irq_ack = ata_irq_ack,
421 .scr_read = nv_scr_read,
422 .scr_write = nv_scr_write,
423 .port_start = ata_port_start,
424 .host_stop = nv_ck804_host_stop,
427 static const struct ata_port_operations nv_adma_ops = {
428 .port_disable = ata_port_disable,
429 .tf_load = ata_tf_load,
430 .tf_read = ata_tf_read,
431 .check_atapi_dma = nv_adma_check_atapi_dma,
432 .exec_command = ata_exec_command,
433 .check_status = ata_check_status,
434 .dev_select = ata_std_dev_select,
435 .bmdma_setup = nv_adma_bmdma_setup,
436 .bmdma_start = nv_adma_bmdma_start,
437 .bmdma_stop = nv_adma_bmdma_stop,
438 .bmdma_status = nv_adma_bmdma_status,
439 .qc_prep = nv_adma_qc_prep,
440 .qc_issue = nv_adma_qc_issue,
441 .freeze = nv_ck804_freeze,
442 .thaw = nv_ck804_thaw,
443 .error_handler = nv_adma_error_handler,
444 .post_internal_cmd = nv_adma_bmdma_stop,
445 .data_xfer = ata_data_xfer,
446 .irq_handler = nv_adma_interrupt,
447 .irq_clear = nv_adma_irq_clear,
448 .irq_on = ata_irq_on,
449 .irq_ack = ata_irq_ack,
450 .scr_read = nv_scr_read,
451 .scr_write = nv_scr_write,
452 .port_start = nv_adma_port_start,
453 .port_stop = nv_adma_port_stop,
454 .port_suspend = nv_adma_port_suspend,
455 .port_resume = nv_adma_port_resume,
456 .host_stop = nv_adma_host_stop,
459 static struct ata_port_info nv_port_info[] = {
463 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
464 ATA_FLAG_HRST_TO_RESUME,
465 .pio_mask = NV_PIO_MASK,
466 .mwdma_mask = NV_MWDMA_MASK,
467 .udma_mask = NV_UDMA_MASK,
468 .port_ops = &nv_generic_ops,
473 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
474 ATA_FLAG_HRST_TO_RESUME,
475 .pio_mask = NV_PIO_MASK,
476 .mwdma_mask = NV_MWDMA_MASK,
477 .udma_mask = NV_UDMA_MASK,
478 .port_ops = &nv_nf2_ops,
483 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
484 ATA_FLAG_HRST_TO_RESUME,
485 .pio_mask = NV_PIO_MASK,
486 .mwdma_mask = NV_MWDMA_MASK,
487 .udma_mask = NV_UDMA_MASK,
488 .port_ops = &nv_ck804_ops,
493 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
494 ATA_FLAG_HRST_TO_RESUME |
495 ATA_FLAG_MMIO | ATA_FLAG_NCQ,
496 .pio_mask = NV_PIO_MASK,
497 .mwdma_mask = NV_MWDMA_MASK,
498 .udma_mask = NV_UDMA_MASK,
499 .port_ops = &nv_adma_ops,
503 MODULE_AUTHOR("NVIDIA");
504 MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
505 MODULE_LICENSE("GPL");
506 MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
507 MODULE_VERSION(DRV_VERSION);
509 static int adma_enabled = 1;
511 static void nv_adma_register_mode(struct ata_port *ap)
513 struct nv_adma_port_priv *pp = ap->private_data;
514 void __iomem *mmio = pp->ctl_block;
517 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
520 tmp = readw(mmio + NV_ADMA_CTL);
521 writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
523 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
526 static void nv_adma_mode(struct ata_port *ap)
528 struct nv_adma_port_priv *pp = ap->private_data;
529 void __iomem *mmio = pp->ctl_block;
532 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
535 WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
537 tmp = readw(mmio + NV_ADMA_CTL);
538 writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
540 pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
543 static int nv_adma_slave_config(struct scsi_device *sdev)
545 struct ata_port *ap = ata_shost_to_port(sdev->host);
546 struct nv_adma_port_priv *pp = ap->private_data;
547 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
549 unsigned long segment_boundary;
550 unsigned short sg_tablesize;
553 u32 current_reg, new_reg, config_mask;
555 rc = ata_scsi_slave_config(sdev);
557 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
558 /* Not a proper libata device, ignore */
561 if (ap->device[sdev->id].class == ATA_DEV_ATAPI) {
563 * NVIDIA reports that ADMA mode does not support ATAPI commands.
564 * Therefore ATAPI commands are sent through the legacy interface.
565 * However, the legacy interface only supports 32-bit DMA.
566 * Restrict DMA parameters as required by the legacy interface
567 * when an ATAPI device is connected.
569 bounce_limit = ATA_DMA_MASK;
570 segment_boundary = ATA_DMA_BOUNDARY;
571 /* Subtract 1 since an extra entry may be needed for padding, see
573 sg_tablesize = LIBATA_MAX_PRD - 1;
575 /* Since the legacy DMA engine is in use, we need to disable ADMA
578 nv_adma_register_mode(ap);
581 bounce_limit = *ap->dev->dma_mask;
582 segment_boundary = NV_ADMA_DMA_BOUNDARY;
583 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
587 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, ¤t_reg);
590 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
591 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
593 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
594 NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
597 new_reg = current_reg | config_mask;
598 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
601 new_reg = current_reg & ~config_mask;
602 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
605 if(current_reg != new_reg)
606 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
608 blk_queue_bounce_limit(sdev->request_queue, bounce_limit);
609 blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
610 blk_queue_max_hw_segments(sdev->request_queue, sg_tablesize);
611 ata_port_printk(ap, KERN_INFO,
612 "bounce limit 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
613 (unsigned long long)bounce_limit, segment_boundary, sg_tablesize);
617 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
619 struct nv_adma_port_priv *pp = qc->ap->private_data;
620 return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
623 static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
625 unsigned int idx = 0;
627 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device | WNB);
629 if ((tf->flags & ATA_TFLAG_LBA48) == 0) {
630 cpb[idx++] = cpu_to_le16(IGN);
631 cpb[idx++] = cpu_to_le16(IGN);
632 cpb[idx++] = cpu_to_le16(IGN);
633 cpb[idx++] = cpu_to_le16(IGN);
634 cpb[idx++] = cpu_to_le16(IGN);
637 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature);
638 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
639 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal);
640 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam);
641 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah);
643 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature);
644 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect);
645 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal);
646 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam);
647 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah);
649 cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND);
654 static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
656 struct nv_adma_port_priv *pp = ap->private_data;
657 u8 flags = pp->cpb[cpb_num].resp_flags;
659 VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags);
661 if (unlikely((force_err ||
662 flags & (NV_CPB_RESP_ATA_ERR |
663 NV_CPB_RESP_CMD_ERR |
664 NV_CPB_RESP_CPB_ERR)))) {
665 struct ata_eh_info *ehi = &ap->eh_info;
668 ata_ehi_clear_desc(ehi);
669 ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x", flags );
670 if (flags & NV_CPB_RESP_ATA_ERR) {
671 ata_ehi_push_desc(ehi, ": ATA error");
672 ehi->err_mask |= AC_ERR_DEV;
673 } else if (flags & NV_CPB_RESP_CMD_ERR) {
674 ata_ehi_push_desc(ehi, ": CMD error");
675 ehi->err_mask |= AC_ERR_DEV;
676 } else if (flags & NV_CPB_RESP_CPB_ERR) {
677 ata_ehi_push_desc(ehi, ": CPB error");
678 ehi->err_mask |= AC_ERR_SYSTEM;
681 /* notifier error, but no error in CPB flags? */
682 ehi->err_mask |= AC_ERR_OTHER;
685 /* Kill all commands. EH will determine what actually failed. */
693 if (flags & NV_CPB_RESP_DONE) {
694 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, cpb_num);
695 VPRINTK("CPB flags done, flags=0x%x\n", flags);
697 /* Grab the ATA port status for non-NCQ commands.
698 For NCQ commands the current status may have nothing to do with
699 the command just completed. */
700 if (qc->tf.protocol != ATA_PROT_NCQ) {
701 u8 ata_status = readb(pp->ctl_block + (ATA_REG_STATUS * 4));
702 qc->err_mask |= ac_err_mask(ata_status);
704 DPRINTK("Completing qc from tag %d with err_mask %u\n",cpb_num,
712 static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
714 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->active_tag);
716 /* freeze if hotplugged */
717 if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
722 /* bail out if not our interrupt */
723 if (!(irq_stat & NV_INT_DEV))
726 /* DEV interrupt w/ no active qc? */
727 if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
728 ata_check_status(ap);
732 /* handle interrupt */
733 return ata_host_intr(ap, qc);
736 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
738 struct ata_host *host = dev_instance;
740 u32 notifier_clears[2];
742 spin_lock(&host->lock);
744 for (i = 0; i < host->n_ports; i++) {
745 struct ata_port *ap = host->ports[i];
746 notifier_clears[i] = 0;
748 if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
749 struct nv_adma_port_priv *pp = ap->private_data;
750 void __iomem *mmio = pp->ctl_block;
753 u32 notifier, notifier_error;
755 /* if in ATA register mode, use standard ata interrupt handler */
756 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
757 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
758 >> (NV_INT_PORT_SHIFT * i);
759 if(ata_tag_valid(ap->active_tag))
760 /** NV_INT_DEV indication seems unreliable at times
761 at least in ADMA mode. Force it on always when a
762 command is active, to prevent losing interrupts. */
763 irq_stat |= NV_INT_DEV;
764 handled += nv_host_intr(ap, irq_stat);
768 notifier = readl(mmio + NV_ADMA_NOTIFIER);
769 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
770 notifier_clears[i] = notifier | notifier_error;
772 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
774 if( !NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
779 status = readw(mmio + NV_ADMA_STAT);
781 /* Clear status. Ensure the controller sees the clearing before we start
782 looking at any of the CPB statuses, so that any CPB completions after
783 this point in the handler will raise another interrupt. */
784 writew(status, mmio + NV_ADMA_STAT);
785 readw(mmio + NV_ADMA_STAT); /* flush posted write */
788 handled++; /* irq handled if we got here */
790 /* freeze if hotplugged or controller error */
791 if (unlikely(status & (NV_ADMA_STAT_HOTPLUG |
792 NV_ADMA_STAT_HOTUNPLUG |
793 NV_ADMA_STAT_TIMEOUT))) {
794 struct ata_eh_info *ehi = &ap->eh_info;
796 ata_ehi_clear_desc(ehi);
797 ata_ehi_push_desc(ehi, "ADMA status 0x%08x", status );
798 if (status & NV_ADMA_STAT_TIMEOUT) {
799 ehi->err_mask |= AC_ERR_SYSTEM;
800 ata_ehi_push_desc(ehi, ": timeout");
801 } else if (status & NV_ADMA_STAT_HOTPLUG) {
802 ata_ehi_hotplugged(ehi);
803 ata_ehi_push_desc(ehi, ": hotplug");
804 } else if (status & NV_ADMA_STAT_HOTUNPLUG) {
805 ata_ehi_hotplugged(ehi);
806 ata_ehi_push_desc(ehi, ": hot unplug");
812 if (status & (NV_ADMA_STAT_DONE |
813 NV_ADMA_STAT_CPBERR)) {
814 /** Check CPBs for completed commands */
816 if (ata_tag_valid(ap->active_tag)) {
817 /* Non-NCQ command */
818 nv_adma_check_cpb(ap, ap->active_tag,
819 notifier_error & (1 << ap->active_tag));
822 u32 active = ap->sactive;
824 while ((pos = ffs(active)) && !error) {
826 error = nv_adma_check_cpb(ap, pos,
827 notifier_error & (1 << pos) );
828 active &= ~(1 << pos );
835 if(notifier_clears[0] || notifier_clears[1]) {
836 /* Note: Both notifier clear registers must be written
837 if either is set, even if one is zero, according to NVIDIA. */
838 struct nv_adma_port_priv *pp = host->ports[0]->private_data;
839 writel(notifier_clears[0], pp->notifier_clear_block);
840 pp = host->ports[1]->private_data;
841 writel(notifier_clears[1], pp->notifier_clear_block);
844 spin_unlock(&host->lock);
846 return IRQ_RETVAL(handled);
849 static void nv_adma_irq_clear(struct ata_port *ap)
851 struct nv_adma_port_priv *pp = ap->private_data;
852 void __iomem *mmio = pp->ctl_block;
853 u16 status = readw(mmio + NV_ADMA_STAT);
854 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
855 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
856 void __iomem *dma_stat_addr = ap->ioaddr.bmdma_addr + ATA_DMA_STATUS;
858 /* clear ADMA status */
859 writew(status, mmio + NV_ADMA_STAT);
860 writel(notifier | notifier_error,
861 pp->notifier_clear_block);
863 /** clear legacy status */
864 iowrite8(ioread8(dma_stat_addr), dma_stat_addr);
867 static void nv_adma_bmdma_setup(struct ata_queued_cmd *qc)
869 struct ata_port *ap = qc->ap;
870 unsigned int rw = (qc->tf.flags & ATA_TFLAG_WRITE);
871 struct nv_adma_port_priv *pp = ap->private_data;
874 if(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
879 /* load PRD table addr. */
880 iowrite32(ap->prd_dma, ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
882 /* specify data direction, triple-check start bit is clear */
883 dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
884 dmactl &= ~(ATA_DMA_WR | ATA_DMA_START);
886 dmactl |= ATA_DMA_WR;
888 iowrite8(dmactl, ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
890 /* issue r/w command */
891 ata_exec_command(ap, &qc->tf);
894 static void nv_adma_bmdma_start(struct ata_queued_cmd *qc)
896 struct ata_port *ap = qc->ap;
897 struct nv_adma_port_priv *pp = ap->private_data;
900 if(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
905 /* start host DMA transaction */
906 dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
907 iowrite8(dmactl | ATA_DMA_START,
908 ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
911 static void nv_adma_bmdma_stop(struct ata_queued_cmd *qc)
913 struct ata_port *ap = qc->ap;
914 struct nv_adma_port_priv *pp = ap->private_data;
916 if(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
919 /* clear start/stop bit */
920 iowrite8(ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD) & ~ATA_DMA_START,
921 ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
923 /* one-PIO-cycle guaranteed wait, per spec, for HDMA1:0 transition */
924 ata_altstatus(ap); /* dummy read */
927 static u8 nv_adma_bmdma_status(struct ata_port *ap)
929 struct nv_adma_port_priv *pp = ap->private_data;
931 WARN_ON(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE));
933 return ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_STATUS);
936 static int nv_adma_port_start(struct ata_port *ap)
938 struct device *dev = ap->host->dev;
939 struct nv_adma_port_priv *pp;
948 rc = ata_port_start(ap);
952 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
956 mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT +
957 ap->port_no * NV_ADMA_PORT_SIZE;
958 pp->ctl_block = mmio;
959 pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;
960 pp->notifier_clear_block = pp->gen_block +
961 NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
963 mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
964 &mem_dma, GFP_KERNEL);
967 memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ);
970 * First item in chunk of DMA memory:
971 * 128-byte command parameter block (CPB)
972 * one for each command tag
975 pp->cpb_dma = mem_dma;
977 writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
978 writel((mem_dma >> 16 ) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
980 mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
981 mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
984 * Second item: block of ADMA_SGTBL_LEN s/g entries
987 pp->aprd_dma = mem_dma;
989 ap->private_data = pp;
991 /* clear any outstanding interrupt conditions */
992 writew(0xffff, mmio + NV_ADMA_STAT);
994 /* initialize port variables */
995 pp->flags = NV_ADMA_PORT_REGISTER_MODE;
997 /* clear CPB fetch count */
998 writew(0, mmio + NV_ADMA_CPB_COUNT);
1000 /* clear GO for register mode, enable interrupt */
1001 tmp = readw(mmio + NV_ADMA_CTL);
1002 writew( (tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN, mmio + NV_ADMA_CTL);
1004 tmp = readw(mmio + NV_ADMA_CTL);
1005 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1006 readl( mmio + NV_ADMA_CTL ); /* flush posted write */
1008 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1009 readl( mmio + NV_ADMA_CTL ); /* flush posted write */
1014 static void nv_adma_port_stop(struct ata_port *ap)
1016 struct nv_adma_port_priv *pp = ap->private_data;
1017 void __iomem *mmio = pp->ctl_block;
1020 writew(0, mmio + NV_ADMA_CTL);
1023 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1025 struct nv_adma_port_priv *pp = ap->private_data;
1026 void __iomem *mmio = pp->ctl_block;
1028 /* Go to register mode - clears GO */
1029 nv_adma_register_mode(ap);
1031 /* clear CPB fetch count */
1032 writew(0, mmio + NV_ADMA_CPB_COUNT);
1034 /* disable interrupt, shut down port */
1035 writew(0, mmio + NV_ADMA_CTL);
1040 static int nv_adma_port_resume(struct ata_port *ap)
1042 struct nv_adma_port_priv *pp = ap->private_data;
1043 void __iomem *mmio = pp->ctl_block;
1046 /* set CPB block location */
1047 writel(pp->cpb_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
1048 writel((pp->cpb_dma >> 16 ) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
1050 /* clear any outstanding interrupt conditions */
1051 writew(0xffff, mmio + NV_ADMA_STAT);
1053 /* initialize port variables */
1054 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1056 /* clear CPB fetch count */
1057 writew(0, mmio + NV_ADMA_CPB_COUNT);
1059 /* clear GO for register mode, enable interrupt */
1060 tmp = readw(mmio + NV_ADMA_CTL);
1061 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN, mmio + NV_ADMA_CTL);
1063 tmp = readw(mmio + NV_ADMA_CTL);
1064 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1065 readl( mmio + NV_ADMA_CTL ); /* flush posted write */
1067 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1068 readl( mmio + NV_ADMA_CTL ); /* flush posted write */
1073 static void nv_adma_setup_port(struct ata_probe_ent *probe_ent, unsigned int port)
1075 void __iomem *mmio = probe_ent->iomap[NV_MMIO_BAR];
1076 struct ata_ioports *ioport = &probe_ent->port[port];
1080 mmio += NV_ADMA_PORT + port * NV_ADMA_PORT_SIZE;
1082 ioport->cmd_addr = mmio;
1083 ioport->data_addr = mmio + (ATA_REG_DATA * 4);
1084 ioport->error_addr =
1085 ioport->feature_addr = mmio + (ATA_REG_ERR * 4);
1086 ioport->nsect_addr = mmio + (ATA_REG_NSECT * 4);
1087 ioport->lbal_addr = mmio + (ATA_REG_LBAL * 4);
1088 ioport->lbam_addr = mmio + (ATA_REG_LBAM * 4);
1089 ioport->lbah_addr = mmio + (ATA_REG_LBAH * 4);
1090 ioport->device_addr = mmio + (ATA_REG_DEVICE * 4);
1091 ioport->status_addr =
1092 ioport->command_addr = mmio + (ATA_REG_STATUS * 4);
1093 ioport->altstatus_addr =
1094 ioport->ctl_addr = mmio + 0x20;
1097 static int nv_adma_host_init(struct ata_probe_ent *probe_ent)
1099 struct pci_dev *pdev = to_pci_dev(probe_ent->dev);
1105 /* enable ADMA on the ports */
1106 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1107 tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1108 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1109 NV_MCP_SATA_CFG_20_PORT1_EN |
1110 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1112 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1114 for (i = 0; i < probe_ent->n_ports; i++)
1115 nv_adma_setup_port(probe_ent, i);
1120 static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1121 struct scatterlist *sg,
1123 struct nv_adma_prd *aprd)
1127 memset(aprd, 0, sizeof(struct nv_adma_prd));
1130 if (qc->tf.flags & ATA_TFLAG_WRITE)
1131 flags |= NV_APRD_WRITE;
1132 if (idx == qc->n_elem - 1)
1133 flags |= NV_APRD_END;
1135 flags |= NV_APRD_CONT;
1137 aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg)));
1138 aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
1139 aprd->flags = flags;
1142 static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1144 struct nv_adma_port_priv *pp = qc->ap->private_data;
1146 struct nv_adma_prd *aprd;
1147 struct scatterlist *sg;
1153 ata_for_each_sg(sg, qc) {
1154 aprd = (idx < 5) ? &cpb->aprd[idx] : &pp->aprd[NV_ADMA_SGTBL_LEN * qc->tag + (idx-5)];
1155 nv_adma_fill_aprd(qc, sg, idx, aprd);
1159 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->tag)));
1162 static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
1164 struct nv_adma_port_priv *pp = qc->ap->private_data;
1165 struct nv_adma_cpb *cpb = &pp->cpb[qc->tag];
1166 u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
1167 NV_CPB_CTL_APRD_VALID |
1170 if (!(qc->flags & ATA_QCFLAG_DMAMAP) ||
1171 (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) {
1172 nv_adma_register_mode(qc->ap);
1177 memset(cpb, 0, sizeof(struct nv_adma_cpb));
1181 cpb->next_cpb_idx = 0;
1183 /* turn on NCQ flags for NCQ commands */
1184 if (qc->tf.protocol == ATA_PROT_NCQ)
1185 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1187 VPRINTK("qc->flags = 0x%lx\n", qc->flags);
1189 nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1191 nv_adma_fill_sg(qc, cpb);
1193 /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID until we are
1194 finished filling in all of the contents */
1196 cpb->ctl_flags = ctl_flags;
1199 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1201 struct nv_adma_port_priv *pp = qc->ap->private_data;
1202 void __iomem *mmio = pp->ctl_block;
1206 if (!(qc->flags & ATA_QCFLAG_DMAMAP) ||
1207 (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) {
1208 /* use ATA register mode */
1209 VPRINTK("no dmamap or ATAPI, using ATA register mode: 0x%lx\n", qc->flags);
1210 nv_adma_register_mode(qc->ap);
1211 return ata_qc_issue_prot(qc);
1213 nv_adma_mode(qc->ap);
1215 /* write append register, command tag in lower 8 bits
1216 and (number of cpbs to append -1) in top 8 bits */
1218 writew(qc->tag, mmio + NV_ADMA_APPEND);
1220 DPRINTK("Issued tag %u\n",qc->tag);
1225 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
1227 struct ata_host *host = dev_instance;
1229 unsigned int handled = 0;
1230 unsigned long flags;
1232 spin_lock_irqsave(&host->lock, flags);
1234 for (i = 0; i < host->n_ports; i++) {
1235 struct ata_port *ap;
1237 ap = host->ports[i];
1239 !(ap->flags & ATA_FLAG_DISABLED)) {
1240 struct ata_queued_cmd *qc;
1242 qc = ata_qc_from_tag(ap, ap->active_tag);
1243 if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING)))
1244 handled += ata_host_intr(ap, qc);
1246 // No request pending? Clear interrupt status
1247 // anyway, in case there's one pending.
1248 ap->ops->check_status(ap);
1253 spin_unlock_irqrestore(&host->lock, flags);
1255 return IRQ_RETVAL(handled);
1258 static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
1262 for (i = 0; i < host->n_ports; i++) {
1263 struct ata_port *ap = host->ports[i];
1265 if (ap && !(ap->flags & ATA_FLAG_DISABLED))
1266 handled += nv_host_intr(ap, irq_stat);
1268 irq_stat >>= NV_INT_PORT_SHIFT;
1271 return IRQ_RETVAL(handled);
1274 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
1276 struct ata_host *host = dev_instance;
1280 spin_lock(&host->lock);
1281 irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
1282 ret = nv_do_interrupt(host, irq_stat);
1283 spin_unlock(&host->lock);
1288 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
1290 struct ata_host *host = dev_instance;
1294 spin_lock(&host->lock);
1295 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1296 ret = nv_do_interrupt(host, irq_stat);
1297 spin_unlock(&host->lock);
1302 static u32 nv_scr_read (struct ata_port *ap, unsigned int sc_reg)
1304 if (sc_reg > SCR_CONTROL)
1307 return ioread32(ap->ioaddr.scr_addr + (sc_reg * 4));
1310 static void nv_scr_write (struct ata_port *ap, unsigned int sc_reg, u32 val)
1312 if (sc_reg > SCR_CONTROL)
1315 iowrite32(val, ap->ioaddr.scr_addr + (sc_reg * 4));
1318 static void nv_nf2_freeze(struct ata_port *ap)
1320 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1321 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1324 mask = ioread8(scr_addr + NV_INT_ENABLE);
1325 mask &= ~(NV_INT_ALL << shift);
1326 iowrite8(mask, scr_addr + NV_INT_ENABLE);
1329 static void nv_nf2_thaw(struct ata_port *ap)
1331 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1332 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1335 iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
1337 mask = ioread8(scr_addr + NV_INT_ENABLE);
1338 mask |= (NV_INT_MASK << shift);
1339 iowrite8(mask, scr_addr + NV_INT_ENABLE);
1342 static void nv_ck804_freeze(struct ata_port *ap)
1344 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1345 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1348 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1349 mask &= ~(NV_INT_ALL << shift);
1350 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1353 static void nv_ck804_thaw(struct ata_port *ap)
1355 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1356 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1359 writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1361 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1362 mask |= (NV_INT_MASK << shift);
1363 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1366 static int nv_hardreset(struct ata_port *ap, unsigned int *class)
1370 /* SATA hardreset fails to retrieve proper device signature on
1371 * some controllers. Don't classify on hardreset. For more
1372 * info, see http://bugme.osdl.org/show_bug.cgi?id=3352
1374 return sata_std_hardreset(ap, &dummy);
1377 static void nv_error_handler(struct ata_port *ap)
1379 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1380 nv_hardreset, ata_std_postreset);
1383 static void nv_adma_error_handler(struct ata_port *ap)
1385 struct nv_adma_port_priv *pp = ap->private_data;
1386 if(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
1387 void __iomem *mmio = pp->ctl_block;
1391 /* Push us back into port register mode for error handling. */
1392 nv_adma_register_mode(ap);
1394 /* Mark all of the CPBs as invalid to prevent them from being executed */
1395 for( i=0;i<NV_ADMA_MAX_CPBS;i++)
1396 pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1398 /* clear CPB fetch count */
1399 writew(0, mmio + NV_ADMA_CPB_COUNT);
1402 tmp = readw(mmio + NV_ADMA_CTL);
1403 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1404 readl( mmio + NV_ADMA_CTL ); /* flush posted write */
1406 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1407 readl( mmio + NV_ADMA_CTL ); /* flush posted write */
1410 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1411 nv_hardreset, ata_std_postreset);
1414 static int nv_init_one (struct pci_dev *pdev, const struct pci_device_id *ent)
1416 static int printed_version = 0;
1417 struct ata_port_info *ppi[2];
1418 struct ata_probe_ent *probe_ent;
1419 struct nv_host_priv *hpriv;
1423 unsigned long type = ent->driver_data;
1426 // Make sure this is a SATA controller by counting the number of bars
1427 // (NVIDIA SATA controllers will always have six bars). Otherwise,
1428 // it's an IDE controller and we ignore it.
1429 for (bar=0; bar<6; bar++)
1430 if (pci_resource_start(pdev, bar) == 0)
1433 if (!printed_version++)
1434 dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
1436 rc = pcim_enable_device(pdev);
1440 rc = pci_request_regions(pdev, DRV_NAME);
1442 pcim_pin_device(pdev);
1446 if(type >= CK804 && adma_enabled) {
1447 dev_printk(KERN_NOTICE, &pdev->dev, "Using ADMA mode\n");
1449 if(!pci_set_dma_mask(pdev, DMA_64BIT_MASK) &&
1450 !pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK))
1455 rc = pci_set_dma_mask(pdev, ATA_DMA_MASK);
1458 rc = pci_set_consistent_dma_mask(pdev, ATA_DMA_MASK);
1465 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
1469 ppi[0] = ppi[1] = &nv_port_info[type];
1470 probe_ent = ata_pci_init_native_mode(pdev, ppi, ATA_PORT_PRIMARY | ATA_PORT_SECONDARY);
1474 if (!pcim_iomap(pdev, NV_MMIO_BAR, 0))
1476 probe_ent->iomap = pcim_iomap_table(pdev);
1478 probe_ent->private_data = hpriv;
1481 base = probe_ent->iomap[NV_MMIO_BAR];
1482 probe_ent->port[0].scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
1483 probe_ent->port[1].scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
1485 /* enable SATA space for CK804 */
1486 if (type >= CK804) {
1489 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, ®val);
1490 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
1491 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
1494 pci_set_master(pdev);
1497 rc = nv_adma_host_init(probe_ent);
1502 rc = ata_device_add(probe_ent);
1506 devm_kfree(&pdev->dev, probe_ent);
1510 static void nv_remove_one (struct pci_dev *pdev)
1512 struct ata_host *host = dev_get_drvdata(&pdev->dev);
1513 struct nv_host_priv *hpriv = host->private_data;
1515 ata_pci_remove_one(pdev);
1519 static int nv_pci_device_resume(struct pci_dev *pdev)
1521 struct ata_host *host = dev_get_drvdata(&pdev->dev);
1522 struct nv_host_priv *hpriv = host->private_data;
1524 ata_pci_device_do_resume(pdev);
1526 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
1527 if(hpriv->type >= CK804) {
1530 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, ®val);
1531 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
1532 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
1534 if(hpriv->type == ADMA) {
1536 struct nv_adma_port_priv *pp;
1537 /* enable/disable ADMA on the ports appropriately */
1538 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1540 pp = host->ports[0]->private_data;
1541 if(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1542 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
1543 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
1545 tmp32 |= (NV_MCP_SATA_CFG_20_PORT0_EN |
1546 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
1547 pp = host->ports[1]->private_data;
1548 if(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1549 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
1550 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
1552 tmp32 |= (NV_MCP_SATA_CFG_20_PORT1_EN |
1553 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
1555 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1559 ata_host_resume(host);
1564 static void nv_ck804_host_stop(struct ata_host *host)
1566 struct pci_dev *pdev = to_pci_dev(host->dev);
1569 /* disable SATA space for CK804 */
1570 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, ®val);
1571 regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
1572 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
1575 static void nv_adma_host_stop(struct ata_host *host)
1577 struct pci_dev *pdev = to_pci_dev(host->dev);
1580 /* disable ADMA on the ports */
1581 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1582 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
1583 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1584 NV_MCP_SATA_CFG_20_PORT1_EN |
1585 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
1587 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1589 nv_ck804_host_stop(host);
1592 static int __init nv_init(void)
1594 return pci_register_driver(&nv_pci_driver);
1597 static void __exit nv_exit(void)
1599 pci_unregister_driver(&nv_pci_driver);
1602 module_init(nv_init);
1603 module_exit(nv_exit);
1604 module_param_named(adma, adma_enabled, bool, 0444);
1605 MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: true)");