The getaddrinfo(3) API does not tell us the TTL value learned from DNS

so we have to add our own stuff for that.

Without some kind of TTL, we would hit the DNS server once per failed
attempt to connect to the backend.

If the backend were down, we could hit it a LOT.

In the VCL code:

	backend foobar {
		[...]
		set backend.dnsttl = 20s;
	}

will assign a TTL for DNS lookups of this backends hostname+port
combination, we will not hit the DNS server more often that this.

The default is set at 30 seconds, short enough to make things are
workable in a load-balancing-via-DNS setups, yet long enough to not
pound the DNS server flat in case of backend failures.

NOTE that as long as we succeed in connecting to the backend we
do not perform new DNS lookups.  That will have to be revisited
along with possible load-balancing schemes for the backend(s).

git-svn-id: svn+ssh://projects.linpro.no/svn/varnish/trunk@1237 d4fa192b-c00b-0410-8231-f00ffab90ce4

diff --git a/varnish-cache/bin/varnishd/cache.h b/varnish-cache/bin/varnishd/cache.h
index 2828a6072b6f1fa4af1878379f3fa474b536d1cd..112354dd04b18ba129543f934681cd6fc6efa71f 100644 (file)
--- a/varnish-cache/bin/varnishd/cache.h
+++ b/varnish-cache/bin/varnishd/cache.h
@@ -302,6 +302,9 @@ struct backend {
        struct addrinfo         *last_addr;
 
        TAILQ_HEAD(,vbe_conn)   connlist;
+
+       double                  dnsttl;
+       double                  dnstime;
 #if 0
        double                  responsetime;
        double                  timeout;

diff --git a/varnish-cache/bin/varnishd/cache_backend.c b/varnish-cache/bin/varnishd/cache_backend.c
index 12c408191a3f6e4963c2482670e2ff15df64f919..7ee57e9242301880b4b5c97def90531f1a1cf71a 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_backend.c
+++ b/varnish-cache/bin/varnishd/cache_backend.c
@@ -58,13 +58,26 @@ static TAILQ_HEAD(,vbe_conn) vbe_head = TAILQ_HEAD_INITIALIZER(vbe_head);
 
 static MTX vbemtx;
 
+/*--------------------------------------------------------------------*/
+/* XXX: belongs a more general place */
+
+static double
+Uptime(void)
+{
+       struct timespec ts;
+
+       assert(clock_gettime(CLOCK_MONOTONIC, &ts) == 0);
+       return (ts.tv_sec + ts.tv_nsec * 1e-9);
+}
+
 /*--------------------------------------------------------------------*/
 
 static struct vbe_conn *
 vbe_new_conn(void)
 {
        struct vbe_conn *vbc;
-       unsigned char *p, space;
+       unsigned char *p;
+       unsigned space;
 
        space =  params->mem_workspace;
        vbc = calloc(sizeof *vbc + space * 2, 1);
@@ -93,6 +106,7 @@ vbe_lookup(struct backend *bp)
        if (bp->addr != NULL) {
                freeaddrinfo(bp->addr);
                bp->addr = NULL;
+               bp->last_addr = NULL;
        }
 
        memset(&hint, 0, sizeof hint);
@@ -102,13 +116,14 @@ vbe_lookup(struct backend *bp)
        error = getaddrinfo(bp->hostname,
            bp->portname == NULL ? "http" : bp->portname,
            &hint, &res);
+       bp->dnstime = Uptime();
        if (error) {
                if (res != NULL)
                        freeaddrinfo(res);
                printf("getaddrinfo: %s\n", gai_strerror(error)); /* XXX */
-               bp->addr = NULL;
                return;
        }
+       bp->last_addr = res;
        bp->addr = res;
 }
 
@@ -120,9 +135,7 @@ vbe_sock_conn(const struct addrinfo *ai)
        int s;
 
        s = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);
-       if (s < 0)
-               return (s);
-       else if (connect(s, ai->ai_addr, ai->ai_addrlen)) {
+       if (s >= 0 && connect(s, ai->ai_addr, ai->ai_addrlen)) {
                AZ(close(s));
                s = -1;
        }
@@ -157,6 +170,9 @@ vbe_conn_try(struct backend *bp, struct addrinfo **pai)
                }
        }
 
+       if (bp->dnstime + bp->dnsttl >= Uptime())
+               return (-1);
+
        /* Then do another lookup to catch DNS changes */
        vbe_lookup(bp);
 

diff --git a/varnish-cache/bin/varnishd/cache_vrt.c b/varnish-cache/bin/varnishd/cache_vrt.c
index 0dffc9b2319028fe181b0370ac8cfaffb8963d19..9b1c0211e564d9ea5858f561354a7592512308f3 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_vrt.c
+++ b/varnish-cache/bin/varnishd/cache_vrt.c
@@ -124,6 +124,7 @@ VRT_alloc_backends(struct VCL_conf *cp)
                cp->backend[i] = calloc(sizeof *cp->backend[i], 1);
                XXXAN(cp->backend[i]);
                cp->backend[i]->magic = BACKEND_MAGIC;
+               cp->backend[i]->dnsttl = 30;
                TAILQ_INIT(&cp->backend[i]->connlist);
        }
 }
@@ -160,6 +161,7 @@ VRT_r_backend_##onm(struct backend *be)                     \
 
 VBACKEND(const char *, host,   hostname)
 VBACKEND(const char *, port,   portname)
+VBACKEND(double,       dnsttl, dnsttl)
 
 /*--------------------------------------------------------------------
  * XXX: Working relative to t_req is maybe not the right thing, we could

diff --git a/varnish-cache/include/vrt_obj.h b/varnish-cache/include/vrt_obj.h
index a82aed93b76b70fc76aa32880141ca70edd22a5b..f0d4fefc7a0e8818076519922003a99288f510b2 100644 (file)
--- a/varnish-cache/include/vrt_obj.h
+++ b/varnish-cache/include/vrt_obj.h
@@ -10,6 +10,8 @@ const char * VRT_r_backend_host(struct backend *);
 void VRT_l_backend_host(struct backend *, const char *);
 const char * VRT_r_backend_port(struct backend *);
 void VRT_l_backend_port(struct backend *, const char *);
+double VRT_r_backend_dnsttl(struct backend *);
+void VRT_l_backend_dnsttl(struct backend *, double);
 const unsigned char * VRT_r_client_ip(struct sess *);
 void VRT_l_client_ip(struct sess *, const unsigned char *);
 const char * VRT_r_req_request(struct sess *);

diff --git a/varnish-cache/lib/libvcl/flint.lnt b/varnish-cache/lib/libvcl/flint.lnt
index a76a92032aa59bab8920a7c56363bcb917b526f1..e1bd648898cfd17aac38d53166d4b0465f2e41c0 100644 (file)
--- a/varnish-cache/lib/libvcl/flint.lnt
+++ b/varnish-cache/lib/libvcl/flint.lnt
@@ -6,6 +6,7 @@
 -printf_code( ju, long long unsigned)
 -printf_code( jx, long long unsigned)
 
++libh ../../config.h
 -header(../../config.h)
 -sem(lbv_assert, r_no)
 -sem(strchr, 1p, type(1), 2n == 0 ? (@p < 1p) : (@p < 1p || @p == 0 ))

diff --git a/varnish-cache/lib/libvcl/vcc_compile.c b/varnish-cache/lib/libvcl/vcc_compile.c
index d5d14f83d53a772e86d6d410047f2c61c374f87d..47587940bedd36b908d2cde8a74f32ae34153c75 100644 (file)
--- a/varnish-cache/lib/libvcl/vcc_compile.c
+++ b/varnish-cache/lib/libvcl/vcc_compile.c
@@ -258,6 +258,7 @@ AddRefStr(struct tokenlist *tl, const char *s, enum ref_type type)
        t->e = strchr(s, '\0');
        t->tok = METHOD;
        AddRef(tl, t, type);
+       /* XXX: possibly leaking t */
 }
 
 void
@@ -931,6 +932,7 @@ CheckHostPort(const char *host, const char *port)
 static void
 Backend(struct tokenlist *tl)
 {
+       unsigned a;
        struct var *vp;
        struct token *t_be = NULL;
        struct token *t_host = NULL;
@@ -984,6 +986,27 @@ Backend(struct tokenlist *tl)
                        Fc(tl, 0, ");\n");
                        vcc_NextToken(tl);
                        break;
+#if 0
+               case INT:
+               case SIZE:
+               case RATE:
+               case FLOAT:
+#endif
+               case TIME:
+                       Fc(tl, 1, "\t%s ", vp->lname);
+                       a = tl->t->tok;
+                       if (a == T_MUL || a == T_DIV)
+                               Fc(tl, 0, "%g", DoubleVal(tl));
+                       else if (vp->fmt == TIME)
+                               TimeVal(tl);
+                       else if (vp->fmt == SIZE)
+                               SizeVal(tl);
+                       else if (vp->fmt == RATE)
+                               RateVal(tl);
+                       else
+                               Fc(tl, 0, "%g", DoubleVal(tl));
+                       Fc(tl, 0, ");\n");
+                       break;
                default:
                        vsb_printf(tl->sb,
                            "Assignments not possible for '%s'\n", vp->name);

diff --git a/varnish-cache/lib/libvcl/vcc_gen_obj.tcl b/varnish-cache/lib/libvcl/vcc_gen_obj.tcl
index a3bb9ddde105ae18696807c21e21a520eec76c8c..24534576f35328d7706b385a189301fc02aad83d 100755 (executable)
--- a/varnish-cache/lib/libvcl/vcc_gen_obj.tcl
+++ b/varnish-cache/lib/libvcl/vcc_gen_obj.tcl
@@ -32,24 +32,25 @@
 
 # Objects which operate on backends
 set beobj {
-       { backend.host  HOSTNAME }
-       { backend.port  PORTNAME }
+       { backend.host          HOSTNAME }
+       { backend.port          PORTNAME }
+       { backend.dnsttl        TIME }
 }
 
 # Objects which operate on sessions
 
 set spobj {
-       { client.ip     IP }
-       { req.request   STRING }
-       { req.host      STRING }
-        { req.url      STRING }
-        { req.proto    STRING }
-        { req.backend  BACKEND }
-        { obj.valid    BOOL }
-        { obj.cacheable        BOOL }
-        { obj.ttl      TIME }
-        { req.http.    HEADER }
-        { resp.http.   HEADER }
+       { client.ip             IP }
+       { req.request           STRING }
+       { req.host              STRING }
+        { req.url              STRING }
+        { req.proto            STRING }
+        { req.backend          BACKEND }
+        { obj.valid            BOOL }
+        { obj.cacheable                BOOL }
+        { obj.ttl              TIME }
+        { req.http.            HEADER }
+        { resp.http.           HEADER }
 }
 
 set tt(IP)     "const unsigned char *"

diff --git a/varnish-cache/lib/libvcl/vcc_obj.c b/varnish-cache/lib/libvcl/vcc_obj.c
index 6b12473dbb4cad603d6b79d305a81ea020a6bf13..dddc17140874ff16d7282ea2e46f21465def5b89 100644 (file)
--- a/varnish-cache/lib/libvcl/vcc_obj.c
+++ b/varnish-cache/lib/libvcl/vcc_obj.c
@@ -18,6 +18,10 @@ struct var vcc_be_vars[] = {
            "VRT_r_backend_port(backend)",
            "VRT_l_backend_port(backend, ",
        },
+       { "backend.dnsttl", TIME, 14,
+           "VRT_r_backend_dnsttl(backend)",
+           "VRT_l_backend_dnsttl(backend, ",
+       },
        { NULL }
 };
 
@@ -82,6 +86,8 @@ const char *vrt_obj_h =
        "void VRT_l_backend_host(struct backend *, const char *);\n"
        "const char * VRT_r_backend_port(struct backend *);\n"
        "void VRT_l_backend_port(struct backend *, const char *);\n"
+       "double VRT_r_backend_dnsttl(struct backend *);\n"
+       "void VRT_l_backend_dnsttl(struct backend *, double);\n"
        "const unsigned char * VRT_r_client_ip(struct sess *);\n"
        "void VRT_l_client_ip(struct sess *, const unsigned char *);\n"
        "const char * VRT_r_req_request(struct sess *);\n"