Added health checks for individual servers within a balanced backend cluster.

The health checks work the same as for a general backend. The worse the health of
an individual server within a balanced backend, the less likely it will be to get new
connections.

git-svn-id: svn+ssh://projects.linpro.no/svn/varnish/trunk@1940 d4fa192b-c00b-0410-8231-f00ffab90ce4

diff --git a/varnish-cache/bin/varnishd/cache.h b/varnish-cache/bin/varnishd/cache.h
index 6e4bdaf9b7943e4a3b6025109cb55a1eb6e22e73..5738bed8a428935a17bd7a9d8d57c42467f8ecae 100644 (file)
--- a/varnish-cache/bin/varnishd/cache.h
+++ b/varnish-cache/bin/varnishd/cache.h
@@ -337,6 +337,7 @@ typedef void vbe_recycle_f(struct worker *w, struct vbe_conn *vc);
 typedef void vbe_init_f(void);
 typedef const char *vbe_gethostname_f(struct backend *);
 typedef void vbe_cleanup_f(struct backend *);
+typedef void vbe_updatehealth_f(struct sess *sp, struct vbe_conn *vc, int);
 
 struct backend_method {
        const char              *name;
@@ -345,6 +346,7 @@ struct backend_method {
        vbe_recycle_f           *recycle;
        vbe_cleanup_f           *cleanup;
        vbe_gethostname_f       *gethostname;
+       vbe_updatehealth_f      *updatehealth;
        vbe_init_f              *init;
 };
 
@@ -396,6 +398,7 @@ void VBE_DropRefLocked(struct backend *);
 struct backend *VBE_NewBackend(struct backend_method *method);
 struct vbe_conn *VBE_NewConn(void);
 void VBE_ReleaseConn(struct vbe_conn *);
+void VBE_UpdateHealth(struct sess *sp, struct vbe_conn *, int);
 
 /* cache_backend_simple.c */
 extern struct backend_method   backend_method_simple;

diff --git a/varnish-cache/bin/varnishd/cache_backend.c b/varnish-cache/bin/varnishd/cache_backend.c
index b810fab0a4bbda5a627f90e7f483e2211dd5d7f9..c7d69081cb548ef34f678f069db0da0e63b87710 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_backend.c
+++ b/varnish-cache/bin/varnishd/cache_backend.c
@@ -231,6 +231,22 @@ VBE_RecycleFd(struct worker *w, struct vbe_conn *vc)
        CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
 }
 
+/* Update health ----------------------------------------------*/
+
+void
+VBE_UpdateHealth(struct sess *sp, struct vbe_conn *vc, int a)
+{
+       struct backend *b;
+
+       CHECK_OBJ_NOTNULL(vc, VBE_CONN_MAGIC);
+       CHECK_OBJ_NOTNULL(vc->backend, BACKEND_MAGIC);
+       b = vc->backend;
+       AN(b->method);
+       AN(b->method->updatehealth);
+       b->method->updatehealth(sp, vc, a);
+       CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+}
+
 /*--------------------------------------------------------------------*/
 
 void

diff --git a/varnish-cache/bin/varnishd/cache_backend_random.c b/varnish-cache/bin/varnishd/cache_backend_random.c
index 5c9c5206f3d64ce30ce2a6ef026cf63f44c32198..50597911b04589d03c6e031aeaba29525b1da126 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_backend_random.c
+++ b/varnish-cache/bin/varnishd/cache_backend_random.c
@@ -50,6 +50,7 @@ struct ber {
        unsigned                magic;
 #define BER_MAGIC              0x645b03f4
        struct brspec           *blist;
+       int                     count;
 #if 0
        /* Store a hash of the backend info given in
         * vcl for comparison when a new vcl file is 
@@ -72,6 +73,7 @@ struct brspec {
        double                  dnstime;
        unsigned                dnsseq;
        TAILQ_HEAD(, vbe_conn)  connlist;
+       int                     health;
 };
 
 /*--------------------------------------------------------------------*/
@@ -217,6 +219,8 @@ ber_nextfd(struct sess *sp)
        struct ber *ber;
        struct brspec *bs;
        double r;
+       int min_health = -10;
+       int num = 0;
 
        CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
        CHECK_OBJ_NOTNULL(sp->backend, BACKEND_MAGIC);
@@ -230,6 +234,19 @@ ber_nextfd(struct sess *sp)
                bs = bs->next;
                CHECK_OBJ_NOTNULL(bs, BRSPEC_MAGIC);
        }
+
+       /* If health is low (bad), use round-robin to find
+        * a server with better health (if possible).
+        */
+       while (bs->health < min_health) {
+               bs = bs->next;
+               num++;
+               if (num > ber->count) {
+                       min_health *= 10;
+                       num = 0;
+               }
+       }       
+
        while (1) {
                LOCK(&bp->mtx);
                vc = TAILQ_FIRST(&bs->connlist);
@@ -378,6 +395,30 @@ ber_GetHostname(struct backend *b)
 
 /*--------------------------------------------------------------------*/
 
+static void
+ber_UpdateHealth(struct sess *sp, struct vbe_conn *vc, int add)
+{
+       struct brspec *bs, *first;
+       struct ber *ber;
+       
+       if (vc != NULL) {
+               CAST_OBJ_NOTNULL(bs, vc->priv, BRSPEC_MAGIC);
+       
+               if (bs->health + add >= -10000 || bs->health + add <= 10000)
+                       bs->health += add;
+       } else {
+               CAST_OBJ_NOTNULL(ber, sp->backend->priv, BRSPEC_MAGIC);
+               first = ber->blist;
+               bs = first;
+               do {
+                       bs = bs->next;
+                       bs->health = (int)((double)bs->health / 2);
+               } while (bs != first);
+       }
+}
+
+/*--------------------------------------------------------------------*/
+
 static void
 ber_Init(void)
 {
@@ -392,6 +433,7 @@ struct backend_method backend_method_random = {
        .close =                ber_ClosedFd,
        .recycle =              ber_RecycleFd,
        .gethostname =          ber_GetHostname,
+       .updatehealth =         ber_UpdateHealth,
        .cleanup =              ber_Cleanup,
        .init =                 ber_Init
 };
@@ -426,6 +468,7 @@ VRT_init_random_backend(struct backend **bp, struct vrt_random_backend *t)
        ber = calloc(sizeof *ber, 1);
        XXXAN(ber);
        ber->magic = BER_MAGIC;
+       ber->count = t->count;
 
        b->priv = ber;
 
@@ -454,6 +497,7 @@ VRT_init_random_backend(struct backend **bp, struct vrt_random_backend *t)
                bs->limit = limit;
                
                bs->dnsttl = 300;
+               bs->health = 0;
                
                if (bs_first == NULL)
                        bs_first = bs;

diff --git a/varnish-cache/bin/varnishd/cache_backend_round_robin.c b/varnish-cache/bin/varnishd/cache_backend_round_robin.c
index 3536f47ecb29333ce1bc8595f8e4409ff07aab98..5ef1404361afe580a444bcc28a0c58e2bfda84c4 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_backend_round_robin.c
+++ b/varnish-cache/bin/varnishd/cache_backend_round_robin.c
@@ -50,6 +50,7 @@ struct brr {
        unsigned                magic;
 #define BRR_MAGIC              0x66f05894
        struct bspec            *blist;
+       int                     count;
 #if 0
        /* Store a hash of the backend info given in
         * vcl for comparison when a new vcl file is 
@@ -71,6 +72,7 @@ struct bspec {
        double                  dnstime;
        unsigned                dnsseq;
        TAILQ_HEAD(, vbe_conn)  connlist;
+       int                     health;
 };
 
 /*--------------------------------------------------------------------*/
@@ -215,13 +217,22 @@ brr_nextfd(struct sess *sp)
        int reuse = 0;
        struct brr *brr;
        struct bspec *bs;
+       int min_health = -10;
+       int num = 0;
 
        CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
        CHECK_OBJ_NOTNULL(sp->backend, BACKEND_MAGIC);
        bp = sp->backend;
        CAST_OBJ_NOTNULL(brr, bp->priv, BRR_MAGIC);
        
-       bs = brr->blist = brr->blist->next;
+       do {
+               bs = brr->blist = brr->blist->next;
+               num++;
+               if (num > brr->count) {
+                       min_health *= 10;
+                       num = 0;
+               }
+       } while (bs->health < min_health);
        
        while (1) {
                LOCK(&bp->mtx);
@@ -371,6 +382,31 @@ brr_GetHostname(struct backend *b)
 
 /*--------------------------------------------------------------------*/
 
+static void
+brr_UpdateHealth(struct sess *sp, struct vbe_conn *vc, int add)
+{
+       struct bspec *bs, *first;
+       struct brr *brr;
+       
+       if (vc != NULL) {
+       
+               CAST_OBJ_NOTNULL(bs, vc->priv, BSPEC_MAGIC);
+       
+               if (bs->health + add >= -10000 || bs->health + add <= 10000)
+                       bs->health += add;
+       } else {
+               CAST_OBJ_NOTNULL(brr, sp->backend->priv, BSPEC_MAGIC);
+               first = brr->blist;
+               bs = first;
+               do {
+                       bs = bs->next;
+                       bs->health = (int)((double)bs->health / 2);
+               } while (bs != first);
+       }
+}
+
+/*--------------------------------------------------------------------*/
+
 static void
 brr_Init(void)
 {
@@ -385,6 +421,7 @@ struct backend_method backend_method_round_robin = {
        .close =                brr_ClosedFd,
        .recycle =              brr_RecycleFd,
        .gethostname =          brr_GetHostname,
+       .updatehealth =         brr_UpdateHealth,
        .cleanup =              brr_Cleanup,
        .init =                 brr_Init
 };
@@ -417,6 +454,7 @@ VRT_init_round_robin_backend(struct backend **bp, struct vrt_round_robin_backend
        brr = calloc(sizeof *brr, 1);
        XXXAN(brr);
        brr->magic = BRR_MAGIC;
+       brr->count = t->count;
 
        b->priv = brr;
 
@@ -437,6 +475,7 @@ VRT_init_round_robin_backend(struct backend **bp, struct vrt_round_robin_backend
                XXXAN(bs->hostname);
                
                bs->dnsttl = 300;
+               bs->health = 0;
                
                if (bs_first == NULL)
                        bs_first = bs;

diff --git a/varnish-cache/bin/varnishd/cache_backend_simple.c b/varnish-cache/bin/varnishd/cache_backend_simple.c
index b4a6b7a98d787a7b5057d8a917c0f895ca87d6c0..0994b425ed980aaa21c55cbe21db61434c689223 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_backend_simple.c
+++ b/varnish-cache/bin/varnishd/cache_backend_simple.c
@@ -347,6 +347,21 @@ bes_GetHostname(struct backend *b)
 
 /*--------------------------------------------------------------------*/
 
+static void
+bes_UpdateHealth(struct sess *sp, struct vbe_conn *vc, int a)
+{
+       (void)sp;
+       (void)vc;
+       (void)a;
+       
+       /* 
+        * Not of any use for simple backend. The global health
+        * parameter of the backend should be enough.
+        */
+}
+
+/*--------------------------------------------------------------------*/
+
 static void
 bes_Init(void)
 {
@@ -361,6 +376,7 @@ struct backend_method backend_method_simple = {
        .close =                bes_ClosedFd,
        .recycle =              bes_RecycleFd,
        .gethostname =          bes_GetHostname,
+       .updatehealth =         bes_UpdateHealth,
        .cleanup =              bes_Cleanup,
        .init =                 bes_Init
 };

diff --git a/varnish-cache/bin/varnishd/cache_center.c b/varnish-cache/bin/varnishd/cache_center.c
index 09f07edca00eaa21c0273fa3b6bf689b3017c86a..3e90146c91150dc0daabb7f2e78c3b25b0669f72 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_center.c
+++ b/varnish-cache/bin/varnishd/cache_center.c
@@ -434,6 +434,7 @@ cnt_hit(struct sess *sp)
        if (minutes > sp->backend->minute_limit) {
                sp->backend->minute_limit++;
                sp->backend->health = (int)((double)sp->backend->health / 2);
+               VBE_UpdateHealth(sp, NULL, 0);
        }
 
        VCL_hit_method(sp);

diff --git a/varnish-cache/bin/varnishd/cache_fetch.c b/varnish-cache/bin/varnishd/cache_fetch.c
index 260f8f1b7d4721e75d3627a7f1dcfbc02a333030..4a9f59ba2778c5691b724efddb16328658018e39 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_fetch.c
+++ b/varnish-cache/bin/varnishd/cache_fetch.c
@@ -377,6 +377,12 @@ Fetch(struct sess *sp)
                cls = 1;
 
        CHECK_OBJ_NOTNULL(sp->backend, BACKEND_MAGIC);
+       
+       if (http_GetStatus(sp->bereq->http) == 200)
+               VBE_UpdateHealth(sp, vc, 1);
+       else if(http_GetStatus(sp->bereq->http) == 504)
+               VBE_UpdateHealth(sp, vc, -1);
+       
        if (cls)
                VBE_ClosedFd(sp->wrk, vc);
        else

diff --git a/varnish-cache/include/vcl.h b/varnish-cache/include/vcl.h
index 7574d3e938d5871bac0a9be79a1c824fc74142a5..b6bb447e67ae9dd90dca6a42ecd4d3b4db582dfc 100644 (file)
--- a/varnish-cache/include/vcl.h
+++ b/varnish-cache/include/vcl.h
@@ -21,7 +21,7 @@ struct VCL_conf {
         struct vrt_ref  *ref;
         unsigned        nref;
         unsigned        busy;
-
+        
        unsigned        nsrc;
        const char      **srcname;
        const char      **srcbody;

diff --git a/varnish-cache/include/vrt.h b/varnish-cache/include/vrt.h
index a54ed181afa1cc5e2cd64ea3ab3976021c7296b7..dc0059c7586741e4f2f1e65706e163f6f257c5d5 100644 (file)
--- a/varnish-cache/include/vrt.h
+++ b/varnish-cache/include/vrt.h
@@ -55,6 +55,7 @@ struct vrt_backend_entry {
 
 struct vrt_round_robin_backend {
        const char      *name;
+       unsigned        count;
        struct vrt_backend_entry *bentry;
 };
 

diff --git a/varnish-cache/lib/libvcl/vcc_backend.c b/varnish-cache/lib/libvcl/vcc_backend.c
index 7fc73372c7be03412cb6ef20dee1a0a72eca56b8..75e58fa663fc8d8f89b71a0168916678c29155e1 100644 (file)
--- a/varnish-cache/lib/libvcl/vcc_backend.c
+++ b/varnish-cache/lib/libvcl/vcc_backend.c
@@ -311,6 +311,7 @@ vcc_ParseBalancedBackend(struct tokenlist *tl)
                        Fc(tl, 0, "\nstatic struct vrt_round_robin_backend sbe_%.*s = {\n",
                            PF(t_be));
                        Fc(tl, 0, "\t.name = \"%.*s\",\n", PF(t_be));
+                       Fc(tl, 0, "\t.count = %d,\n", cnt);
                        Fc(tl, 0, "\t.bentry = &bentry_%.*s_%d\n", PF(t_be), cnt-1);
                        Fc(tl, 0, "};\n");
                        Fi(tl, 0, "\tVRT_init_round_robin_backend(&VGC_backend_%.*s , &sbe_%.*s);\n",

diff --git a/varnish-cache/lib/libvcl/vcc_fixed_token.c b/varnish-cache/lib/libvcl/vcc_fixed_token.c
index f9400e5cf4c400f45025e6149d904264ffb0e235..7d63b3b27862f5ad2f401d570eb42b1f1cc2fb16 100644 (file)
--- a/varnish-cache/lib/libvcl/vcc_fixed_token.c
+++ b/varnish-cache/lib/libvcl/vcc_fixed_token.c
@@ -433,6 +433,7 @@ vcl_output_lang_h(struct vsb *sb)
        vsb_cat(sb, "\n");
        vsb_cat(sb, "struct vrt_round_robin_backend {\n");
        vsb_cat(sb, "   const char      *name;\n");
+       vsb_cat(sb, "   unsigned        count;\n");
        vsb_cat(sb, "   struct vrt_backend_entry *bentry;\n");
        vsb_cat(sb, "};\n");
        vsb_cat(sb, "\n");