2 * sata_nv.c - NVIDIA nForce SATA
4 * Copyright 2004 NVIDIA Corp. All rights reserved.
5 * Copyright 2004 Andrew Chew
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program; see the file COPYING. If not, write to
20 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
23 * libata documentation is available via 'make {ps|pdf}docs',
24 * as Documentation/DocBook/libata.*
26 * No hardware documentation available outside of NVIDIA.
27 * This driver programs the NVIDIA SATA controller in a similar
28 * fashion as with other PCI IDE BMDMA controllers, with a few
29 * NV-specific details such as register offsets, SATA phy location,
32 * CK804/MCP04 controllers support an alternate programming interface
33 * similar to the ADMA specification (with some modifications).
34 * This allows the use of NCQ. Non-DMA-mapped ATA commands are still
35 * sent through the legacy interface.
39 #include <linux/kernel.h>
40 #include <linux/module.h>
41 #include <linux/pci.h>
42 #include <linux/init.h>
43 #include <linux/blkdev.h>
44 #include <linux/delay.h>
45 #include <linux/interrupt.h>
46 #include <linux/device.h>
47 #include <scsi/scsi_host.h>
48 #include <scsi/scsi_device.h>
49 #include <linux/libata.h>
51 #define DRV_NAME "sata_nv"
52 #define DRV_VERSION "3.3"
54 #define NV_ADMA_DMA_BOUNDARY 0xffffffffUL
61 NV_PORT0_SCR_REG_OFFSET = 0x00,
62 NV_PORT1_SCR_REG_OFFSET = 0x40,
64 /* INT_STATUS/ENABLE */
67 NV_INT_STATUS_CK804 = 0x440,
68 NV_INT_ENABLE_CK804 = 0x441,
70 /* INT_STATUS/ENABLE bits */
74 NV_INT_REMOVED = 0x08,
76 NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */
79 NV_INT_MASK = NV_INT_DEV |
80 NV_INT_ADDED | NV_INT_REMOVED,
84 NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI
86 // For PCI config register 20
87 NV_MCP_SATA_CFG_20 = 0x50,
88 NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
89 NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17),
90 NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16),
91 NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14),
92 NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12),
94 NV_ADMA_MAX_CPBS = 32,
97 NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) /
99 NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5,
100 NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
101 NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS *
102 (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
104 /* BAR5 offset to ADMA general registers */
106 NV_ADMA_GEN_CTL = 0x00,
107 NV_ADMA_NOTIFIER_CLEAR = 0x30,
109 /* BAR5 offset to ADMA ports */
110 NV_ADMA_PORT = 0x480,
112 /* size of ADMA port register space */
113 NV_ADMA_PORT_SIZE = 0x100,
115 /* ADMA port registers */
117 NV_ADMA_CPB_COUNT = 0x42,
118 NV_ADMA_NEXT_CPB_IDX = 0x43,
120 NV_ADMA_CPB_BASE_LOW = 0x48,
121 NV_ADMA_CPB_BASE_HIGH = 0x4C,
122 NV_ADMA_APPEND = 0x50,
123 NV_ADMA_NOTIFIER = 0x68,
124 NV_ADMA_NOTIFIER_ERROR = 0x6C,
126 /* NV_ADMA_CTL register bits */
127 NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0),
128 NV_ADMA_CTL_CHANNEL_RESET = (1 << 5),
129 NV_ADMA_CTL_GO = (1 << 7),
130 NV_ADMA_CTL_AIEN = (1 << 8),
131 NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11),
132 NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12),
134 /* CPB response flag bits */
135 NV_CPB_RESP_DONE = (1 << 0),
136 NV_CPB_RESP_ATA_ERR = (1 << 3),
137 NV_CPB_RESP_CMD_ERR = (1 << 4),
138 NV_CPB_RESP_CPB_ERR = (1 << 7),
140 /* CPB control flag bits */
141 NV_CPB_CTL_CPB_VALID = (1 << 0),
142 NV_CPB_CTL_QUEUE = (1 << 1),
143 NV_CPB_CTL_APRD_VALID = (1 << 2),
144 NV_CPB_CTL_IEN = (1 << 3),
145 NV_CPB_CTL_FPDMA = (1 << 4),
148 NV_APRD_WRITE = (1 << 1),
149 NV_APRD_END = (1 << 2),
150 NV_APRD_CONT = (1 << 3),
152 /* NV_ADMA_STAT flags */
153 NV_ADMA_STAT_TIMEOUT = (1 << 0),
154 NV_ADMA_STAT_HOTUNPLUG = (1 << 1),
155 NV_ADMA_STAT_HOTPLUG = (1 << 2),
156 NV_ADMA_STAT_CPBERR = (1 << 4),
157 NV_ADMA_STAT_SERROR = (1 << 5),
158 NV_ADMA_STAT_CMD_COMPLETE = (1 << 6),
159 NV_ADMA_STAT_IDLE = (1 << 8),
160 NV_ADMA_STAT_LEGACY = (1 << 9),
161 NV_ADMA_STAT_STOPPED = (1 << 10),
162 NV_ADMA_STAT_DONE = (1 << 12),
163 NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR |
164 NV_ADMA_STAT_TIMEOUT,
167 NV_ADMA_PORT_REGISTER_MODE = (1 << 0),
168 NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1),
172 /* ADMA Physical Region Descriptor - one SG segment */
181 enum nv_adma_regbits {
182 CMDEND = (1 << 15), /* end of command list */
183 WNB = (1 << 14), /* wait-not-BSY */
184 IGN = (1 << 13), /* ignore this entry */
185 CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */
186 DA2 = (1 << (2 + 8)),
187 DA1 = (1 << (1 + 8)),
188 DA0 = (1 << (0 + 8)),
191 /* ADMA Command Parameter Block
192 The first 5 SG segments are stored inside the Command Parameter Block itself.
193 If there are more than 5 segments the remainder are stored in a separate
194 memory area indicated by next_aprd. */
196 u8 resp_flags; /* 0 */
197 u8 reserved1; /* 1 */
198 u8 ctl_flags; /* 2 */
199 /* len is length of taskfile in 64 bit words */
202 u8 next_cpb_idx; /* 5 */
203 __le16 reserved2; /* 6-7 */
204 __le16 tf[12]; /* 8-31 */
205 struct nv_adma_prd aprd[5]; /* 32-111 */
206 __le64 next_aprd; /* 112-119 */
207 __le64 reserved3; /* 120-127 */
211 struct nv_adma_port_priv {
212 struct nv_adma_cpb *cpb;
214 struct nv_adma_prd *aprd;
216 void __iomem * ctl_block;
217 void __iomem * gen_block;
218 void __iomem * notifier_clear_block;
222 struct nv_host_priv {
226 #define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & ( 1 << (19 + (12 * (PORT)))))
228 static int nv_init_one (struct pci_dev *pdev, const struct pci_device_id *ent);
229 static void nv_remove_one (struct pci_dev *pdev);
230 static int nv_pci_device_resume(struct pci_dev *pdev);
231 static void nv_ck804_host_stop(struct ata_host *host);
232 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
233 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
234 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
235 static u32 nv_scr_read (struct ata_port *ap, unsigned int sc_reg);
236 static void nv_scr_write (struct ata_port *ap, unsigned int sc_reg, u32 val);
238 static void nv_nf2_freeze(struct ata_port *ap);
239 static void nv_nf2_thaw(struct ata_port *ap);
240 static void nv_ck804_freeze(struct ata_port *ap);
241 static void nv_ck804_thaw(struct ata_port *ap);
242 static void nv_error_handler(struct ata_port *ap);
243 static int nv_adma_slave_config(struct scsi_device *sdev);
244 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
245 static void nv_adma_qc_prep(struct ata_queued_cmd *qc);
246 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
247 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
248 static void nv_adma_irq_clear(struct ata_port *ap);
249 static int nv_adma_port_start(struct ata_port *ap);
250 static void nv_adma_port_stop(struct ata_port *ap);
251 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
252 static int nv_adma_port_resume(struct ata_port *ap);
253 static void nv_adma_error_handler(struct ata_port *ap);
254 static void nv_adma_host_stop(struct ata_host *host);
255 static void nv_adma_bmdma_setup(struct ata_queued_cmd *qc);
256 static void nv_adma_bmdma_start(struct ata_queued_cmd *qc);
257 static void nv_adma_bmdma_stop(struct ata_queued_cmd *qc);
258 static u8 nv_adma_bmdma_status(struct ata_port *ap);
264 NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */
269 static const struct pci_device_id nv_pci_tbl[] = {
270 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 },
271 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 },
272 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 },
273 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 },
274 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 },
275 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 },
276 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 },
277 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), GENERIC },
278 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), GENERIC },
279 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), GENERIC },
280 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), GENERIC },
281 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC },
282 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC },
283 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC },
284 { PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID,
285 PCI_ANY_ID, PCI_ANY_ID,
286 PCI_CLASS_STORAGE_IDE<<8, 0xffff00, GENERIC },
287 { PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID,
288 PCI_ANY_ID, PCI_ANY_ID,
289 PCI_CLASS_STORAGE_RAID<<8, 0xffff00, GENERIC },
291 { } /* terminate list */
294 static struct pci_driver nv_pci_driver = {
296 .id_table = nv_pci_tbl,
297 .probe = nv_init_one,
298 .suspend = ata_pci_device_suspend,
299 .resume = nv_pci_device_resume,
300 .remove = nv_remove_one,
303 static struct scsi_host_template nv_sht = {
304 .module = THIS_MODULE,
306 .ioctl = ata_scsi_ioctl,
307 .queuecommand = ata_scsi_queuecmd,
308 .can_queue = ATA_DEF_QUEUE,
309 .this_id = ATA_SHT_THIS_ID,
310 .sg_tablesize = LIBATA_MAX_PRD,
311 .cmd_per_lun = ATA_SHT_CMD_PER_LUN,
312 .emulated = ATA_SHT_EMULATED,
313 .use_clustering = ATA_SHT_USE_CLUSTERING,
314 .proc_name = DRV_NAME,
315 .dma_boundary = ATA_DMA_BOUNDARY,
316 .slave_configure = ata_scsi_slave_config,
317 .slave_destroy = ata_scsi_slave_destroy,
318 .bios_param = ata_std_bios_param,
319 .suspend = ata_scsi_device_suspend,
320 .resume = ata_scsi_device_resume,
323 static struct scsi_host_template nv_adma_sht = {
324 .module = THIS_MODULE,
326 .ioctl = ata_scsi_ioctl,
327 .queuecommand = ata_scsi_queuecmd,
328 .can_queue = NV_ADMA_MAX_CPBS,
329 .this_id = ATA_SHT_THIS_ID,
330 .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN,
331 .cmd_per_lun = ATA_SHT_CMD_PER_LUN,
332 .emulated = ATA_SHT_EMULATED,
333 .use_clustering = ATA_SHT_USE_CLUSTERING,
334 .proc_name = DRV_NAME,
335 .dma_boundary = NV_ADMA_DMA_BOUNDARY,
336 .slave_configure = nv_adma_slave_config,
337 .slave_destroy = ata_scsi_slave_destroy,
338 .bios_param = ata_std_bios_param,
339 .suspend = ata_scsi_device_suspend,
340 .resume = ata_scsi_device_resume,
343 static const struct ata_port_operations nv_generic_ops = {
344 .port_disable = ata_port_disable,
345 .tf_load = ata_tf_load,
346 .tf_read = ata_tf_read,
347 .exec_command = ata_exec_command,
348 .check_status = ata_check_status,
349 .dev_select = ata_std_dev_select,
350 .bmdma_setup = ata_bmdma_setup,
351 .bmdma_start = ata_bmdma_start,
352 .bmdma_stop = ata_bmdma_stop,
353 .bmdma_status = ata_bmdma_status,
354 .qc_prep = ata_qc_prep,
355 .qc_issue = ata_qc_issue_prot,
356 .freeze = ata_bmdma_freeze,
357 .thaw = ata_bmdma_thaw,
358 .error_handler = nv_error_handler,
359 .post_internal_cmd = ata_bmdma_post_internal_cmd,
360 .data_xfer = ata_pio_data_xfer,
361 .irq_handler = nv_generic_interrupt,
362 .irq_clear = ata_bmdma_irq_clear,
363 .scr_read = nv_scr_read,
364 .scr_write = nv_scr_write,
365 .port_start = ata_port_start,
366 .port_stop = ata_port_stop,
367 .host_stop = ata_pci_host_stop,
370 static const struct ata_port_operations nv_nf2_ops = {
371 .port_disable = ata_port_disable,
372 .tf_load = ata_tf_load,
373 .tf_read = ata_tf_read,
374 .exec_command = ata_exec_command,
375 .check_status = ata_check_status,
376 .dev_select = ata_std_dev_select,
377 .bmdma_setup = ata_bmdma_setup,
378 .bmdma_start = ata_bmdma_start,
379 .bmdma_stop = ata_bmdma_stop,
380 .bmdma_status = ata_bmdma_status,
381 .qc_prep = ata_qc_prep,
382 .qc_issue = ata_qc_issue_prot,
383 .freeze = nv_nf2_freeze,
385 .error_handler = nv_error_handler,
386 .post_internal_cmd = ata_bmdma_post_internal_cmd,
387 .data_xfer = ata_pio_data_xfer,
388 .irq_handler = nv_nf2_interrupt,
389 .irq_clear = ata_bmdma_irq_clear,
390 .scr_read = nv_scr_read,
391 .scr_write = nv_scr_write,
392 .port_start = ata_port_start,
393 .port_stop = ata_port_stop,
394 .host_stop = ata_pci_host_stop,
397 static const struct ata_port_operations nv_ck804_ops = {
398 .port_disable = ata_port_disable,
399 .tf_load = ata_tf_load,
400 .tf_read = ata_tf_read,
401 .exec_command = ata_exec_command,
402 .check_status = ata_check_status,
403 .dev_select = ata_std_dev_select,
404 .bmdma_setup = ata_bmdma_setup,
405 .bmdma_start = ata_bmdma_start,
406 .bmdma_stop = ata_bmdma_stop,
407 .bmdma_status = ata_bmdma_status,
408 .qc_prep = ata_qc_prep,
409 .qc_issue = ata_qc_issue_prot,
410 .freeze = nv_ck804_freeze,
411 .thaw = nv_ck804_thaw,
412 .error_handler = nv_error_handler,
413 .post_internal_cmd = ata_bmdma_post_internal_cmd,
414 .data_xfer = ata_pio_data_xfer,
415 .irq_handler = nv_ck804_interrupt,
416 .irq_clear = ata_bmdma_irq_clear,
417 .scr_read = nv_scr_read,
418 .scr_write = nv_scr_write,
419 .port_start = ata_port_start,
420 .port_stop = ata_port_stop,
421 .host_stop = nv_ck804_host_stop,
424 static const struct ata_port_operations nv_adma_ops = {
425 .port_disable = ata_port_disable,
426 .tf_load = ata_tf_load,
427 .tf_read = ata_tf_read,
428 .check_atapi_dma = nv_adma_check_atapi_dma,
429 .exec_command = ata_exec_command,
430 .check_status = ata_check_status,
431 .dev_select = ata_std_dev_select,
432 .bmdma_setup = nv_adma_bmdma_setup,
433 .bmdma_start = nv_adma_bmdma_start,
434 .bmdma_stop = nv_adma_bmdma_stop,
435 .bmdma_status = nv_adma_bmdma_status,
436 .qc_prep = nv_adma_qc_prep,
437 .qc_issue = nv_adma_qc_issue,
438 .freeze = nv_ck804_freeze,
439 .thaw = nv_ck804_thaw,
440 .error_handler = nv_adma_error_handler,
441 .post_internal_cmd = nv_adma_bmdma_stop,
442 .data_xfer = ata_mmio_data_xfer,
443 .irq_handler = nv_adma_interrupt,
444 .irq_clear = nv_adma_irq_clear,
445 .scr_read = nv_scr_read,
446 .scr_write = nv_scr_write,
447 .port_start = nv_adma_port_start,
448 .port_stop = nv_adma_port_stop,
449 .port_suspend = nv_adma_port_suspend,
450 .port_resume = nv_adma_port_resume,
451 .host_stop = nv_adma_host_stop,
454 static struct ata_port_info nv_port_info[] = {
458 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
459 ATA_FLAG_HRST_TO_RESUME,
460 .pio_mask = NV_PIO_MASK,
461 .mwdma_mask = NV_MWDMA_MASK,
462 .udma_mask = NV_UDMA_MASK,
463 .port_ops = &nv_generic_ops,
468 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
469 ATA_FLAG_HRST_TO_RESUME,
470 .pio_mask = NV_PIO_MASK,
471 .mwdma_mask = NV_MWDMA_MASK,
472 .udma_mask = NV_UDMA_MASK,
473 .port_ops = &nv_nf2_ops,
478 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
479 ATA_FLAG_HRST_TO_RESUME,
480 .pio_mask = NV_PIO_MASK,
481 .mwdma_mask = NV_MWDMA_MASK,
482 .udma_mask = NV_UDMA_MASK,
483 .port_ops = &nv_ck804_ops,
488 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
489 ATA_FLAG_HRST_TO_RESUME |
490 ATA_FLAG_MMIO | ATA_FLAG_NCQ,
491 .pio_mask = NV_PIO_MASK,
492 .mwdma_mask = NV_MWDMA_MASK,
493 .udma_mask = NV_UDMA_MASK,
494 .port_ops = &nv_adma_ops,
498 MODULE_AUTHOR("NVIDIA");
499 MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
500 MODULE_LICENSE("GPL");
501 MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
502 MODULE_VERSION(DRV_VERSION);
504 static int adma_enabled = 1;
506 static void nv_adma_register_mode(struct ata_port *ap)
508 struct nv_adma_port_priv *pp = ap->private_data;
509 void __iomem *mmio = pp->ctl_block;
512 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
515 tmp = readw(mmio + NV_ADMA_CTL);
516 writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
518 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
521 static void nv_adma_mode(struct ata_port *ap)
523 struct nv_adma_port_priv *pp = ap->private_data;
524 void __iomem *mmio = pp->ctl_block;
527 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
530 WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
532 tmp = readw(mmio + NV_ADMA_CTL);
533 writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
535 pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
538 static int nv_adma_slave_config(struct scsi_device *sdev)
540 struct ata_port *ap = ata_shost_to_port(sdev->host);
541 struct nv_adma_port_priv *pp = ap->private_data;
542 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
544 unsigned long segment_boundary;
545 unsigned short sg_tablesize;
548 u32 current_reg, new_reg, config_mask;
550 rc = ata_scsi_slave_config(sdev);
552 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
553 /* Not a proper libata device, ignore */
556 if (ap->device[sdev->id].class == ATA_DEV_ATAPI) {
558 * NVIDIA reports that ADMA mode does not support ATAPI commands.
559 * Therefore ATAPI commands are sent through the legacy interface.
560 * However, the legacy interface only supports 32-bit DMA.
561 * Restrict DMA parameters as required by the legacy interface
562 * when an ATAPI device is connected.
564 bounce_limit = ATA_DMA_MASK;
565 segment_boundary = ATA_DMA_BOUNDARY;
566 /* Subtract 1 since an extra entry may be needed for padding, see
568 sg_tablesize = LIBATA_MAX_PRD - 1;
570 /* Since the legacy DMA engine is in use, we need to disable ADMA
573 nv_adma_register_mode(ap);
576 bounce_limit = *ap->dev->dma_mask;
577 segment_boundary = NV_ADMA_DMA_BOUNDARY;
578 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
582 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, ¤t_reg);
585 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
586 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
588 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
589 NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
592 new_reg = current_reg | config_mask;
593 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
596 new_reg = current_reg & ~config_mask;
597 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
600 if(current_reg != new_reg)
601 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
603 blk_queue_bounce_limit(sdev->request_queue, bounce_limit);
604 blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
605 blk_queue_max_hw_segments(sdev->request_queue, sg_tablesize);
606 ata_port_printk(ap, KERN_INFO,
607 "bounce limit 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
608 (unsigned long long)bounce_limit, segment_boundary, sg_tablesize);
612 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
614 struct nv_adma_port_priv *pp = qc->ap->private_data;
615 return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
618 static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
620 unsigned int idx = 0;
622 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device | WNB);
624 if ((tf->flags & ATA_TFLAG_LBA48) == 0) {
625 cpb[idx++] = cpu_to_le16(IGN);
626 cpb[idx++] = cpu_to_le16(IGN);
627 cpb[idx++] = cpu_to_le16(IGN);
628 cpb[idx++] = cpu_to_le16(IGN);
629 cpb[idx++] = cpu_to_le16(IGN);
632 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature);
633 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
634 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal);
635 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam);
636 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah);
638 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature);
639 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect);
640 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal);
641 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam);
642 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah);
644 cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND);
649 static void nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
651 struct nv_adma_port_priv *pp = ap->private_data;
652 int complete = 0, have_err = 0;
653 u8 flags = pp->cpb[cpb_num].resp_flags;
655 VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags);
657 if (flags & NV_CPB_RESP_DONE) {
658 VPRINTK("CPB flags done, flags=0x%x\n", flags);
661 if (flags & NV_CPB_RESP_ATA_ERR) {
662 ata_port_printk(ap, KERN_ERR, "CPB flags ATA err, flags=0x%x\n", flags);
666 if (flags & NV_CPB_RESP_CMD_ERR) {
667 ata_port_printk(ap, KERN_ERR, "CPB flags CMD err, flags=0x%x\n", flags);
671 if (flags & NV_CPB_RESP_CPB_ERR) {
672 ata_port_printk(ap, KERN_ERR, "CPB flags CPB err, flags=0x%x\n", flags);
676 if(complete || force_err)
678 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, cpb_num);
681 /* Only use the ATA port status for non-NCQ commands.
682 For NCQ commands the current status may have nothing to do with
683 the command just completed. */
684 if(qc->tf.protocol != ATA_PROT_NCQ)
685 ata_status = readb(pp->ctl_block + (ATA_REG_STATUS * 4));
687 if(have_err || force_err)
688 ata_status |= ATA_ERR;
690 qc->err_mask |= ac_err_mask(ata_status);
691 DPRINTK("Completing qc from tag %d with err_mask %u\n",cpb_num,
698 static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
700 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->active_tag);
702 /* freeze if hotplugged */
703 if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
708 /* bail out if not our interrupt */
709 if (!(irq_stat & NV_INT_DEV))
712 /* DEV interrupt w/ no active qc? */
713 if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
714 ata_check_status(ap);
718 /* handle interrupt */
719 return ata_host_intr(ap, qc);
722 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
724 struct ata_host *host = dev_instance;
726 u32 notifier_clears[2];
728 spin_lock(&host->lock);
730 for (i = 0; i < host->n_ports; i++) {
731 struct ata_port *ap = host->ports[i];
732 notifier_clears[i] = 0;
734 if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
735 struct nv_adma_port_priv *pp = ap->private_data;
736 void __iomem *mmio = pp->ctl_block;
739 int have_global_err = 0;
740 u32 notifier, notifier_error;
742 /* if in ATA register mode, use standard ata interrupt handler */
743 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
744 u8 irq_stat = readb(host->mmio_base + NV_INT_STATUS_CK804)
745 >> (NV_INT_PORT_SHIFT * i);
746 if(ata_tag_valid(ap->active_tag))
747 /** NV_INT_DEV indication seems unreliable at times
748 at least in ADMA mode. Force it on always when a
749 command is active, to prevent losing interrupts. */
750 irq_stat |= NV_INT_DEV;
751 handled += nv_host_intr(ap, irq_stat);
755 notifier = readl(mmio + NV_ADMA_NOTIFIER);
756 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
757 notifier_clears[i] = notifier | notifier_error;
759 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
761 if( !NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
766 status = readw(mmio + NV_ADMA_STAT);
768 /* Clear status. Ensure the controller sees the clearing before we start
769 looking at any of the CPB statuses, so that any CPB completions after
770 this point in the handler will raise another interrupt. */
771 writew(status, mmio + NV_ADMA_STAT);
772 readw(mmio + NV_ADMA_STAT); /* flush posted write */
775 /* freeze if hotplugged */
776 if (unlikely(status & (NV_ADMA_STAT_HOTPLUG | NV_ADMA_STAT_HOTUNPLUG))) {
777 ata_port_printk(ap, KERN_NOTICE, "Hotplug event, freezing\n");
783 if (status & NV_ADMA_STAT_TIMEOUT) {
784 ata_port_printk(ap, KERN_ERR, "timeout, stat=0x%x\n", status);
787 if (status & NV_ADMA_STAT_CPBERR) {
788 ata_port_printk(ap, KERN_ERR, "CPB error, stat=0x%x\n", status);
791 if ((status & NV_ADMA_STAT_DONE) || have_global_err) {
792 /** Check CPBs for completed commands */
794 if(ata_tag_valid(ap->active_tag))
795 /* Non-NCQ command */
796 nv_adma_check_cpb(ap, ap->active_tag, have_global_err ||
797 (notifier_error & (1 << ap->active_tag)));
800 u32 active = ap->sactive;
801 while( (pos = ffs(active)) ) {
803 nv_adma_check_cpb(ap, pos, have_global_err ||
804 (notifier_error & (1 << pos)) );
805 active &= ~(1 << pos );
810 handled++; /* irq handled if we got here */
814 if(notifier_clears[0] || notifier_clears[1]) {
815 /* Note: Both notifier clear registers must be written
816 if either is set, even if one is zero, according to NVIDIA. */
817 struct nv_adma_port_priv *pp = host->ports[0]->private_data;
818 writel(notifier_clears[0], pp->notifier_clear_block);
819 pp = host->ports[1]->private_data;
820 writel(notifier_clears[1], pp->notifier_clear_block);
823 spin_unlock(&host->lock);
825 return IRQ_RETVAL(handled);
828 static void nv_adma_irq_clear(struct ata_port *ap)
830 struct nv_adma_port_priv *pp = ap->private_data;
831 void __iomem *mmio = pp->ctl_block;
832 u16 status = readw(mmio + NV_ADMA_STAT);
833 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
834 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
835 unsigned long dma_stat_addr = ap->ioaddr.bmdma_addr + ATA_DMA_STATUS;
837 /* clear ADMA status */
838 writew(status, mmio + NV_ADMA_STAT);
839 writel(notifier | notifier_error,
840 pp->notifier_clear_block);
842 /** clear legacy status */
843 outb(inb(dma_stat_addr), dma_stat_addr);
846 static void nv_adma_bmdma_setup(struct ata_queued_cmd *qc)
848 struct ata_port *ap = qc->ap;
849 unsigned int rw = (qc->tf.flags & ATA_TFLAG_WRITE);
850 struct nv_adma_port_priv *pp = ap->private_data;
853 if(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
858 /* load PRD table addr. */
859 outl(ap->prd_dma, ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
861 /* specify data direction, triple-check start bit is clear */
862 dmactl = inb(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
863 dmactl &= ~(ATA_DMA_WR | ATA_DMA_START);
865 dmactl |= ATA_DMA_WR;
867 outb(dmactl, ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
869 /* issue r/w command */
870 ata_exec_command(ap, &qc->tf);
873 static void nv_adma_bmdma_start(struct ata_queued_cmd *qc)
875 struct ata_port *ap = qc->ap;
876 struct nv_adma_port_priv *pp = ap->private_data;
879 if(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
884 /* start host DMA transaction */
885 dmactl = inb(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
886 outb(dmactl | ATA_DMA_START,
887 ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
890 static void nv_adma_bmdma_stop(struct ata_queued_cmd *qc)
892 struct ata_port *ap = qc->ap;
893 struct nv_adma_port_priv *pp = ap->private_data;
895 if(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
898 /* clear start/stop bit */
899 outb(inb(ap->ioaddr.bmdma_addr + ATA_DMA_CMD) & ~ATA_DMA_START,
900 ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
902 /* one-PIO-cycle guaranteed wait, per spec, for HDMA1:0 transition */
903 ata_altstatus(ap); /* dummy read */
906 static u8 nv_adma_bmdma_status(struct ata_port *ap)
908 struct nv_adma_port_priv *pp = ap->private_data;
910 WARN_ON(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE));
912 return inb(ap->ioaddr.bmdma_addr + ATA_DMA_STATUS);
915 static int nv_adma_port_start(struct ata_port *ap)
917 struct device *dev = ap->host->dev;
918 struct nv_adma_port_priv *pp;
927 rc = ata_port_start(ap);
931 pp = kzalloc(sizeof(*pp), GFP_KERNEL);
937 mmio = ap->host->mmio_base + NV_ADMA_PORT +
938 ap->port_no * NV_ADMA_PORT_SIZE;
939 pp->ctl_block = mmio;
940 pp->gen_block = ap->host->mmio_base + NV_ADMA_GEN;
941 pp->notifier_clear_block = pp->gen_block +
942 NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
944 mem = dma_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
945 &mem_dma, GFP_KERNEL);
951 memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ);
954 * First item in chunk of DMA memory:
955 * 128-byte command parameter block (CPB)
956 * one for each command tag
959 pp->cpb_dma = mem_dma;
961 writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
962 writel((mem_dma >> 16 ) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
964 mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
965 mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
968 * Second item: block of ADMA_SGTBL_LEN s/g entries
971 pp->aprd_dma = mem_dma;
973 ap->private_data = pp;
975 /* clear any outstanding interrupt conditions */
976 writew(0xffff, mmio + NV_ADMA_STAT);
978 /* initialize port variables */
979 pp->flags = NV_ADMA_PORT_REGISTER_MODE;
981 /* clear CPB fetch count */
982 writew(0, mmio + NV_ADMA_CPB_COUNT);
984 /* clear GO for register mode, enable interrupt */
985 tmp = readw(mmio + NV_ADMA_CTL);
986 writew( (tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN, mmio + NV_ADMA_CTL);
988 tmp = readw(mmio + NV_ADMA_CTL);
989 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
990 readl( mmio + NV_ADMA_CTL ); /* flush posted write */
992 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
993 readl( mmio + NV_ADMA_CTL ); /* flush posted write */
1004 static void nv_adma_port_stop(struct ata_port *ap)
1006 struct device *dev = ap->host->dev;
1007 struct nv_adma_port_priv *pp = ap->private_data;
1008 void __iomem *mmio = pp->ctl_block;
1012 writew(0, mmio + NV_ADMA_CTL);
1014 ap->private_data = NULL;
1015 dma_free_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ, pp->cpb, pp->cpb_dma);
1020 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1022 struct nv_adma_port_priv *pp = ap->private_data;
1023 void __iomem *mmio = pp->ctl_block;
1025 /* Go to register mode - clears GO */
1026 nv_adma_register_mode(ap);
1028 /* clear CPB fetch count */
1029 writew(0, mmio + NV_ADMA_CPB_COUNT);
1031 /* disable interrupt, shut down port */
1032 writew(0, mmio + NV_ADMA_CTL);
1037 static int nv_adma_port_resume(struct ata_port *ap)
1039 struct nv_adma_port_priv *pp = ap->private_data;
1040 void __iomem *mmio = pp->ctl_block;
1043 /* set CPB block location */
1044 writel(pp->cpb_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
1045 writel((pp->cpb_dma >> 16 ) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
1047 /* clear any outstanding interrupt conditions */
1048 writew(0xffff, mmio + NV_ADMA_STAT);
1050 /* initialize port variables */
1051 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1053 /* clear CPB fetch count */
1054 writew(0, mmio + NV_ADMA_CPB_COUNT);
1056 /* clear GO for register mode, enable interrupt */
1057 tmp = readw(mmio + NV_ADMA_CTL);
1058 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN, mmio + NV_ADMA_CTL);
1060 tmp = readw(mmio + NV_ADMA_CTL);
1061 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1062 readl( mmio + NV_ADMA_CTL ); /* flush posted write */
1064 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1065 readl( mmio + NV_ADMA_CTL ); /* flush posted write */
1070 static void nv_adma_setup_port(struct ata_probe_ent *probe_ent, unsigned int port)
1072 void __iomem *mmio = probe_ent->mmio_base;
1073 struct ata_ioports *ioport = &probe_ent->port[port];
1077 mmio += NV_ADMA_PORT + port * NV_ADMA_PORT_SIZE;
1079 ioport->cmd_addr = (unsigned long) mmio;
1080 ioport->data_addr = (unsigned long) mmio + (ATA_REG_DATA * 4);
1081 ioport->error_addr =
1082 ioport->feature_addr = (unsigned long) mmio + (ATA_REG_ERR * 4);
1083 ioport->nsect_addr = (unsigned long) mmio + (ATA_REG_NSECT * 4);
1084 ioport->lbal_addr = (unsigned long) mmio + (ATA_REG_LBAL * 4);
1085 ioport->lbam_addr = (unsigned long) mmio + (ATA_REG_LBAM * 4);
1086 ioport->lbah_addr = (unsigned long) mmio + (ATA_REG_LBAH * 4);
1087 ioport->device_addr = (unsigned long) mmio + (ATA_REG_DEVICE * 4);
1088 ioport->status_addr =
1089 ioport->command_addr = (unsigned long) mmio + (ATA_REG_STATUS * 4);
1090 ioport->altstatus_addr =
1091 ioport->ctl_addr = (unsigned long) mmio + 0x20;
1094 static int nv_adma_host_init(struct ata_probe_ent *probe_ent)
1096 struct pci_dev *pdev = to_pci_dev(probe_ent->dev);
1102 /* enable ADMA on the ports */
1103 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1104 tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1105 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1106 NV_MCP_SATA_CFG_20_PORT1_EN |
1107 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1109 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1111 for (i = 0; i < probe_ent->n_ports; i++)
1112 nv_adma_setup_port(probe_ent, i);
1117 static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1118 struct scatterlist *sg,
1120 struct nv_adma_prd *aprd)
1124 memset(aprd, 0, sizeof(struct nv_adma_prd));
1127 if (qc->tf.flags & ATA_TFLAG_WRITE)
1128 flags |= NV_APRD_WRITE;
1129 if (idx == qc->n_elem - 1)
1130 flags |= NV_APRD_END;
1132 flags |= NV_APRD_CONT;
1134 aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg)));
1135 aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
1136 aprd->flags = flags;
1139 static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1141 struct nv_adma_port_priv *pp = qc->ap->private_data;
1143 struct nv_adma_prd *aprd;
1144 struct scatterlist *sg;
1150 ata_for_each_sg(sg, qc) {
1151 aprd = (idx < 5) ? &cpb->aprd[idx] : &pp->aprd[NV_ADMA_SGTBL_LEN * qc->tag + (idx-5)];
1152 nv_adma_fill_aprd(qc, sg, idx, aprd);
1156 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->tag)));
1159 static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
1161 struct nv_adma_port_priv *pp = qc->ap->private_data;
1162 struct nv_adma_cpb *cpb = &pp->cpb[qc->tag];
1163 u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
1164 NV_CPB_CTL_APRD_VALID |
1167 if (!(qc->flags & ATA_QCFLAG_DMAMAP) ||
1168 (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) {
1169 nv_adma_register_mode(qc->ap);
1174 memset(cpb, 0, sizeof(struct nv_adma_cpb));
1178 cpb->next_cpb_idx = 0;
1180 /* turn on NCQ flags for NCQ commands */
1181 if (qc->tf.protocol == ATA_PROT_NCQ)
1182 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1184 VPRINTK("qc->flags = 0x%lx\n", qc->flags);
1186 nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1188 nv_adma_fill_sg(qc, cpb);
1190 /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID until we are
1191 finished filling in all of the contents */
1193 cpb->ctl_flags = ctl_flags;
1196 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1198 struct nv_adma_port_priv *pp = qc->ap->private_data;
1199 void __iomem *mmio = pp->ctl_block;
1203 if (!(qc->flags & ATA_QCFLAG_DMAMAP) ||
1204 (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) {
1205 /* use ATA register mode */
1206 VPRINTK("no dmamap or ATAPI, using ATA register mode: 0x%lx\n", qc->flags);
1207 nv_adma_register_mode(qc->ap);
1208 return ata_qc_issue_prot(qc);
1210 nv_adma_mode(qc->ap);
1212 /* write append register, command tag in lower 8 bits
1213 and (number of cpbs to append -1) in top 8 bits */
1215 writew(qc->tag, mmio + NV_ADMA_APPEND);
1217 DPRINTK("Issued tag %u\n",qc->tag);
1222 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
1224 struct ata_host *host = dev_instance;
1226 unsigned int handled = 0;
1227 unsigned long flags;
1229 spin_lock_irqsave(&host->lock, flags);
1231 for (i = 0; i < host->n_ports; i++) {
1232 struct ata_port *ap;
1234 ap = host->ports[i];
1236 !(ap->flags & ATA_FLAG_DISABLED)) {
1237 struct ata_queued_cmd *qc;
1239 qc = ata_qc_from_tag(ap, ap->active_tag);
1240 if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING)))
1241 handled += ata_host_intr(ap, qc);
1243 // No request pending? Clear interrupt status
1244 // anyway, in case there's one pending.
1245 ap->ops->check_status(ap);
1250 spin_unlock_irqrestore(&host->lock, flags);
1252 return IRQ_RETVAL(handled);
1255 static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
1259 for (i = 0; i < host->n_ports; i++) {
1260 struct ata_port *ap = host->ports[i];
1262 if (ap && !(ap->flags & ATA_FLAG_DISABLED))
1263 handled += nv_host_intr(ap, irq_stat);
1265 irq_stat >>= NV_INT_PORT_SHIFT;
1268 return IRQ_RETVAL(handled);
1271 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
1273 struct ata_host *host = dev_instance;
1277 spin_lock(&host->lock);
1278 irq_stat = inb(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
1279 ret = nv_do_interrupt(host, irq_stat);
1280 spin_unlock(&host->lock);
1285 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
1287 struct ata_host *host = dev_instance;
1291 spin_lock(&host->lock);
1292 irq_stat = readb(host->mmio_base + NV_INT_STATUS_CK804);
1293 ret = nv_do_interrupt(host, irq_stat);
1294 spin_unlock(&host->lock);
1299 static u32 nv_scr_read (struct ata_port *ap, unsigned int sc_reg)
1301 if (sc_reg > SCR_CONTROL)
1304 return ioread32((void __iomem *)ap->ioaddr.scr_addr + (sc_reg * 4));
1307 static void nv_scr_write (struct ata_port *ap, unsigned int sc_reg, u32 val)
1309 if (sc_reg > SCR_CONTROL)
1312 iowrite32(val, (void __iomem *)ap->ioaddr.scr_addr + (sc_reg * 4));
1315 static void nv_nf2_freeze(struct ata_port *ap)
1317 unsigned long scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1318 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1321 mask = inb(scr_addr + NV_INT_ENABLE);
1322 mask &= ~(NV_INT_ALL << shift);
1323 outb(mask, scr_addr + NV_INT_ENABLE);
1326 static void nv_nf2_thaw(struct ata_port *ap)
1328 unsigned long scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1329 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1332 outb(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
1334 mask = inb(scr_addr + NV_INT_ENABLE);
1335 mask |= (NV_INT_MASK << shift);
1336 outb(mask, scr_addr + NV_INT_ENABLE);
1339 static void nv_ck804_freeze(struct ata_port *ap)
1341 void __iomem *mmio_base = ap->host->mmio_base;
1342 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1345 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1346 mask &= ~(NV_INT_ALL << shift);
1347 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1350 static void nv_ck804_thaw(struct ata_port *ap)
1352 void __iomem *mmio_base = ap->host->mmio_base;
1353 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1356 writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1358 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1359 mask |= (NV_INT_MASK << shift);
1360 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1363 static int nv_hardreset(struct ata_port *ap, unsigned int *class)
1367 /* SATA hardreset fails to retrieve proper device signature on
1368 * some controllers. Don't classify on hardreset. For more
1369 * info, see http://bugme.osdl.org/show_bug.cgi?id=3352
1371 return sata_std_hardreset(ap, &dummy);
1374 static void nv_error_handler(struct ata_port *ap)
1376 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1377 nv_hardreset, ata_std_postreset);
1380 static void nv_adma_error_handler(struct ata_port *ap)
1382 struct nv_adma_port_priv *pp = ap->private_data;
1383 if(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
1384 void __iomem *mmio = pp->ctl_block;
1388 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1389 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1390 u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1391 u32 status = readw(mmio + NV_ADMA_STAT);
1393 ata_port_printk(ap, KERN_ERR, "EH in ADMA mode, notifier 0x%X "
1394 "notifier_error 0x%X gen_ctl 0x%X status 0x%X\n",
1395 notifier, notifier_error, gen_ctl, status);
1397 for( i=0;i<NV_ADMA_MAX_CPBS;i++) {
1398 struct nv_adma_cpb *cpb = &pp->cpb[i];
1399 if( cpb->ctl_flags || cpb->resp_flags )
1400 ata_port_printk(ap, KERN_ERR,
1401 "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n",
1402 i, cpb->ctl_flags, cpb->resp_flags);
1405 /* Push us back into port register mode for error handling. */
1406 nv_adma_register_mode(ap);
1408 ata_port_printk(ap, KERN_ERR, "Resetting port\n");
1410 /* Mark all of the CPBs as invalid to prevent them from being executed */
1411 for( i=0;i<NV_ADMA_MAX_CPBS;i++)
1412 pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1414 /* clear CPB fetch count */
1415 writew(0, mmio + NV_ADMA_CPB_COUNT);
1418 tmp = readw(mmio + NV_ADMA_CTL);
1419 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1420 readl( mmio + NV_ADMA_CTL ); /* flush posted write */
1422 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1423 readl( mmio + NV_ADMA_CTL ); /* flush posted write */
1426 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1427 nv_hardreset, ata_std_postreset);
1430 static int nv_init_one (struct pci_dev *pdev, const struct pci_device_id *ent)
1432 static int printed_version = 0;
1433 struct ata_port_info *ppi[2];
1434 struct ata_probe_ent *probe_ent;
1435 struct nv_host_priv *hpriv;
1436 int pci_dev_busy = 0;
1440 unsigned long type = ent->driver_data;
1443 // Make sure this is a SATA controller by counting the number of bars
1444 // (NVIDIA SATA controllers will always have six bars). Otherwise,
1445 // it's an IDE controller and we ignore it.
1446 for (bar=0; bar<6; bar++)
1447 if (pci_resource_start(pdev, bar) == 0)
1450 if (!printed_version++)
1451 dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
1453 rc = pci_enable_device(pdev);
1457 rc = pci_request_regions(pdev, DRV_NAME);
1460 goto err_out_disable;
1463 if(type >= CK804 && adma_enabled) {
1464 dev_printk(KERN_NOTICE, &pdev->dev, "Using ADMA mode\n");
1466 if(!pci_set_dma_mask(pdev, DMA_64BIT_MASK) &&
1467 !pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK))
1472 rc = pci_set_dma_mask(pdev, ATA_DMA_MASK);
1474 goto err_out_regions;
1475 rc = pci_set_consistent_dma_mask(pdev, ATA_DMA_MASK);
1477 goto err_out_regions;
1482 hpriv = kmalloc(sizeof(*hpriv), GFP_KERNEL);
1484 goto err_out_regions;
1486 ppi[0] = ppi[1] = &nv_port_info[type];
1487 probe_ent = ata_pci_init_native_mode(pdev, ppi, ATA_PORT_PRIMARY | ATA_PORT_SECONDARY);
1489 goto err_out_regions;
1491 probe_ent->mmio_base = pci_iomap(pdev, 5, 0);
1492 if (!probe_ent->mmio_base) {
1494 goto err_out_free_ent;
1496 probe_ent->private_data = hpriv;
1499 base = (unsigned long)probe_ent->mmio_base;
1501 probe_ent->port[0].scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
1502 probe_ent->port[1].scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
1504 /* enable SATA space for CK804 */
1505 if (type >= CK804) {
1508 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, ®val);
1509 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
1510 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
1513 pci_set_master(pdev);
1516 rc = nv_adma_host_init(probe_ent);
1518 goto err_out_iounmap;
1521 rc = ata_device_add(probe_ent);
1523 goto err_out_iounmap;
1530 pci_iounmap(pdev, probe_ent->mmio_base);
1534 pci_release_regions(pdev);
1537 pci_disable_device(pdev);
1542 static void nv_remove_one (struct pci_dev *pdev)
1544 struct ata_host *host = dev_get_drvdata(&pdev->dev);
1545 struct nv_host_priv *hpriv = host->private_data;
1547 ata_pci_remove_one(pdev);
1551 static int nv_pci_device_resume(struct pci_dev *pdev)
1553 struct ata_host *host = dev_get_drvdata(&pdev->dev);
1554 struct nv_host_priv *hpriv = host->private_data;
1556 ata_pci_device_do_resume(pdev);
1558 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
1559 if(hpriv->type >= CK804) {
1562 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, ®val);
1563 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
1564 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
1566 if(hpriv->type == ADMA) {
1568 struct nv_adma_port_priv *pp;
1569 /* enable/disable ADMA on the ports appropriately */
1570 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1572 pp = host->ports[0]->private_data;
1573 if(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1574 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
1575 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
1577 tmp32 |= (NV_MCP_SATA_CFG_20_PORT0_EN |
1578 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
1579 pp = host->ports[1]->private_data;
1580 if(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1581 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
1582 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
1584 tmp32 |= (NV_MCP_SATA_CFG_20_PORT1_EN |
1585 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
1587 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1591 ata_host_resume(host);
1596 static void nv_ck804_host_stop(struct ata_host *host)
1598 struct pci_dev *pdev = to_pci_dev(host->dev);
1601 /* disable SATA space for CK804 */
1602 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, ®val);
1603 regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
1604 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
1606 ata_pci_host_stop(host);
1609 static void nv_adma_host_stop(struct ata_host *host)
1611 struct pci_dev *pdev = to_pci_dev(host->dev);
1614 /* disable ADMA on the ports */
1615 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1616 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
1617 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1618 NV_MCP_SATA_CFG_20_PORT1_EN |
1619 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
1621 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1623 nv_ck804_host_stop(host);
1626 static int __init nv_init(void)
1628 return pci_register_driver(&nv_pci_driver);
1631 static void __exit nv_exit(void)
1633 pci_unregister_driver(&nv_pci_driver);
1636 module_init(nv_init);
1637 module_exit(nv_exit);
1638 module_param_named(adma, adma_enabled, bool, 0444);
1639 MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: true)");