IB/ehca: Refactor sync between completions and destroy_cq using atomic_t

- ehca_cq.nr_events is made an atomic_t, eliminating a lot of locking.
- The CQ is removed from the CQ idr first now to make sure no more
  completions are scheduled on that CQ. The "wait for all completions to
  end" code becomes much simpler this way.

Signed-off-by: Joachim Fenkes <fenkes@de.ibm.com>
Signed-off-by: Roland Dreier <rolandd@cisco.com>

diff --git a/drivers/infiniband/hw/ehca/ehca_classes.h b/drivers/infiniband/hw/ehca/ehca_classes.h
index 3550047c13759017ab72e392c72b2b3a69afc630..8580f2a0ea57dab2a4168c7c606306cea448cbf8 100644 (file)
--- a/drivers/infiniband/hw/ehca/ehca_classes.h
+++ b/drivers/infiniband/hw/ehca/ehca_classes.h
@@ -174,8 +174,8 @@ struct ehca_cq {
        spinlock_t cb_lock;
        struct hlist_head qp_hashtab[QP_HASHTAB_LEN];
        struct list_head entry;
-       u32 nr_callbacks; /* #events assigned to cpu by scaling code */
-       u32 nr_events;    /* #events seen */
+       u32 nr_callbacks;   /* #events assigned to cpu by scaling code */
+       atomic_t nr_events; /* #events seen */
        wait_queue_head_t wait_completion;
        spinlock_t task_lock;
        u32 ownpid;

diff --git a/drivers/infiniband/hw/ehca/ehca_cq.c b/drivers/infiniband/hw/ehca/ehca_cq.c
index 94bad273b34c7da4b8c479bdf5c70411af4d0074..3729997457caae4dd1963d840b9d76773afbb249 100644 (file)
--- a/drivers/infiniband/hw/ehca/ehca_cq.c
+++ b/drivers/infiniband/hw/ehca/ehca_cq.c
@@ -146,6 +146,7 @@ struct ib_cq *ehca_create_cq(struct ib_device *device, int cqe, int comp_vector,
        spin_lock_init(&my_cq->spinlock);
        spin_lock_init(&my_cq->cb_lock);
        spin_lock_init(&my_cq->task_lock);
+       atomic_set(&my_cq->nr_events, 0);
        init_waitqueue_head(&my_cq->wait_completion);
        my_cq->ownpid = current->tgid;
 
@@ -303,16 +304,6 @@ create_cq_exit1:
        return cq;
 }
 
-static int get_cq_nr_events(struct ehca_cq *my_cq)
-{
-       int ret;
-       unsigned long flags;
-       spin_lock_irqsave(&ehca_cq_idr_lock, flags);
-       ret = my_cq->nr_events;
-       spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);
-       return ret;
-}
-
 int ehca_destroy_cq(struct ib_cq *cq)
 {
        u64 h_ret;
@@ -339,17 +330,18 @@ int ehca_destroy_cq(struct ib_cq *cq)
                }
        }
 
+       /*
+        * remove the CQ from the idr first to make sure
+        * no more interrupt tasklets will touch this CQ
+        */
        spin_lock_irqsave(&ehca_cq_idr_lock, flags);
-       while (my_cq->nr_events) {
-               spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);
-               wait_event(my_cq->wait_completion, !get_cq_nr_events(my_cq));
-               spin_lock_irqsave(&ehca_cq_idr_lock, flags);
-               /* recheck nr_events to assure no cqe has just arrived */
-       }
-
        idr_remove(&ehca_cq_idr, my_cq->token);
        spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);
 
+       /* now wait until all pending events have completed */
+       wait_event(my_cq->wait_completion, !atomic_read(&my_cq->nr_events));
+
+       /* nobody's using our CQ any longer -- we can destroy it */
        h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 0);
        if (h_ret == H_R_STATE) {
                /* cq in err: read err data and destroy it forcibly */

diff --git a/drivers/infiniband/hw/ehca/ehca_irq.c b/drivers/infiniband/hw/ehca/ehca_irq.c
index 100329ba33439f79c30c391e17c9e677dbe8c979..3e790a326d97601cef26978f0df2309349b14264 100644 (file)
--- a/drivers/infiniband/hw/ehca/ehca_irq.c
+++ b/drivers/infiniband/hw/ehca/ehca_irq.c
@@ -5,6 +5,8 @@
  *
  *  Authors: Heiko J Schick <schickhj@de.ibm.com>
  *           Khadija Souissi <souissi@de.ibm.com>
+ *           Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *           Joachim Fenkes <fenkes@de.ibm.com>
  *
  *  Copyright (c) 2005 IBM Corporation
  *
@@ -212,6 +214,8 @@ static void cq_event_callback(struct ehca_shca *shca,
 
        spin_lock_irqsave(&ehca_cq_idr_lock, flags);
        cq = idr_find(&ehca_cq_idr, token);
+       if (cq)
+               atomic_inc(&cq->nr_events);
        spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);
 
        if (!cq)
@@ -219,6 +223,9 @@ static void cq_event_callback(struct ehca_shca *shca,
 
        ehca_error_data(shca, cq, cq->ipz_cq_handle.handle);
 
+       if (atomic_dec_and_test(&cq->nr_events))
+               wake_up(&cq->wait_completion);
+
        return;
 }
 
@@ -414,25 +421,22 @@ static inline void process_eqe(struct ehca_shca *shca, struct ehca_eqe *eqe)
                token = EHCA_BMASK_GET(EQE_CQ_TOKEN, eqe_value);
                spin_lock_irqsave(&ehca_cq_idr_lock, flags);
                cq = idr_find(&ehca_cq_idr, token);
+               if (cq)
+                       atomic_inc(&cq->nr_events);
+               spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);
                if (cq == NULL) {
-                       spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);
                        ehca_err(&shca->ib_device,
                                 "Invalid eqe for non-existing cq token=%x",
                                 token);
                        return;
                }
                reset_eq_pending(cq);
-               cq->nr_events++;
-               spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);
                if (ehca_scaling_code)
                        queue_comp_task(cq);
                else {
                        comp_event_callback(cq);
-                       spin_lock_irqsave(&ehca_cq_idr_lock, flags);
-                       cq->nr_events--;
-                       if (!cq->nr_events)
+                       if (atomic_dec_and_test(&cq->nr_events))
                                wake_up(&cq->wait_completion);
-                       spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);
                }
        } else {
                ehca_dbg(&shca->ib_device, "Got non completion event");
@@ -478,15 +482,15 @@ void ehca_process_eq(struct ehca_shca *shca, int is_irq)
                        token = EHCA_BMASK_GET(EQE_CQ_TOKEN, eqe_value);
                        spin_lock(&ehca_cq_idr_lock);
                        eqe_cache[eqe_cnt].cq = idr_find(&ehca_cq_idr, token);
+                       if (eqe_cache[eqe_cnt].cq)
+                               atomic_inc(&eqe_cache[eqe_cnt].cq->nr_events);
+                       spin_unlock(&ehca_cq_idr_lock);
                        if (!eqe_cache[eqe_cnt].cq) {
-                               spin_unlock(&ehca_cq_idr_lock);
                                ehca_err(&shca->ib_device,
                                         "Invalid eqe for non-existing cq "
                                         "token=%x", token);
                                continue;
                        }
-                       eqe_cache[eqe_cnt].cq->nr_events++;
-                       spin_unlock(&ehca_cq_idr_lock);
                } else
                        eqe_cache[eqe_cnt].cq = NULL;
                eqe_cnt++;
@@ -517,11 +521,8 @@ void ehca_process_eq(struct ehca_shca *shca, int is_irq)
                        else {
                                struct ehca_cq *cq = eq->eqe_cache[i].cq;
                                comp_event_callback(cq);
-                               spin_lock(&ehca_cq_idr_lock);
-                               cq->nr_events--;
-                               if (!cq->nr_events)
+                               if (atomic_dec_and_test(&cq->nr_events))
                                        wake_up(&cq->wait_completion);
-                               spin_unlock(&ehca_cq_idr_lock);
                        }
                } else {
                        ehca_dbg(&shca->ib_device, "Got non completion event");
@@ -621,13 +622,10 @@ static void run_comp_task(struct ehca_cpu_comp_task* cct)
        while (!list_empty(&cct->cq_list)) {
                cq = list_entry(cct->cq_list.next, struct ehca_cq, entry);
                spin_unlock_irqrestore(&cct->task_lock, flags);
-               comp_event_callback(cq);
 
-               spin_lock_irqsave(&ehca_cq_idr_lock, flags);
-               cq->nr_events--;
-               if (!cq->nr_events)
+               comp_event_callback(cq);
+               if (atomic_dec_and_test(&cq->nr_events))
                        wake_up(&cq->wait_completion);
-               spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);
 
                spin_lock_irqsave(&cct->task_lock, flags);
                spin_lock(&cq->task_lock);

diff --git a/drivers/infiniband/hw/ehca/ehca_irq.h b/drivers/infiniband/hw/ehca/ehca_irq.h
index 6ed06ee033ed2f326a6674d21701eed3479fa2df..3346cb06cea62a4d4a4036364fba3cf174756dda 100644 (file)
--- a/drivers/infiniband/hw/ehca/ehca_irq.h
+++ b/drivers/infiniband/hw/ehca/ehca_irq.h
@@ -47,7 +47,6 @@ struct ehca_shca;
 
 #include <linux/interrupt.h>
 #include <linux/types.h>
-#include <asm/atomic.h>
 
 int ehca_error_data(struct ehca_shca *shca, void *data, u64 resource);
 

diff --git a/drivers/infiniband/hw/ehca/ehca_tools.h b/drivers/infiniband/hw/ehca/ehca_tools.h
index 973c4b59154575fa9a0dad89864051183d848bc8..03b185f873dafb8c8641358efef02f9f1633dd0f 100644 (file)
--- a/drivers/infiniband/hw/ehca/ehca_tools.h
+++ b/drivers/infiniband/hw/ehca/ehca_tools.h
@@ -59,6 +59,7 @@
 #include <linux/cpu.h>
 #include <linux/device.h>
 
+#include <asm/atomic.h>
 #include <asm/abs_addr.h>
 #include <asm/ibmebus.h>
 #include <asm/io.h>