From: phk Date: Wed, 2 Aug 2006 11:58:54 +0000 (+0000) Subject: Remove the libevent from the backend pool manager. X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a23fea0706a261f91fb2dac727c624b16b54a546;p=varnish Remove the libevent from the backend pool manager. Simplify the logic here while we're at it. git-svn-id: svn+ssh://projects.linpro.no/svn/varnish/trunk@602 d4fa192b-c00b-0410-8231-f00ffab90ce4 --- diff --git a/varnish-cache/bin/varnishd/cache.h b/varnish-cache/bin/varnishd/cache.h index 1c9c8b80..e5f89eef 100644 --- a/varnish-cache/bin/varnishd/cache.h +++ b/varnish-cache/bin/varnishd/cache.h @@ -8,7 +8,6 @@ #include #include "queue.h" -#include "event.h" #include "sbuf.h" #include "vcl_returns.h" @@ -121,10 +120,8 @@ struct vbe_conn { unsigned magic; #define VBE_CONN_MAGIC 0x0c5e6592 TAILQ_ENTRY(vbe_conn) list; - struct vbe *vbe; + struct backend *backend; int fd; - struct event ev; - int inuse; struct http *http; struct http *http2; struct http http_mem[2]; @@ -263,22 +260,22 @@ struct sess { struct backend { unsigned magic; #define BACKEND_MAGIC 0x64c4c7c6 - const char *vcl_name; - const char *hostname; - const char *portname; - unsigned ip; + const char *vcl_name; + const char *hostname; + const char *portname; + unsigned ip; + + struct addrinfo *addr; + struct addrinfo *last_addr; - struct addrinfo *addr; - struct addrinfo *last_addr; + TAILQ_HEAD(,vbe_conn) connlist; #if 0 - double responsetime; - double timeout; - double bandwidth; - int down; + double responsetime; + double timeout; + double bandwidth; + int down; #endif - /* internal stuff */ - struct vbe *vbe; }; /* Prototypes etc ----------------------------------------------------*/ diff --git a/varnish-cache/bin/varnishd/cache_backend.c b/varnish-cache/bin/varnishd/cache_backend.c index e5a040d5..5879c00a 100644 --- a/varnish-cache/bin/varnishd/cache_backend.c +++ b/varnish-cache/bin/varnishd/cache_backend.c @@ -3,23 +3,9 @@ * * Manage backend connections. * - * For each backend ip number we maintain a shadow backend structure so - * that backend connections can be reused across VCL changes. - * - * For each IP we maintain a list of busy and free backend connections, - * and free connections are monitored to detect if the backend closes - * the connection. - * - * We recycle backend connections in most recently used order to minimize - * the number of open connections to any one backend. - * - * XXX: - * I'm not happy about recycling always going through the monitor thread - * but not doing so is slightly more tricky: A connection might be reused - * several times before the monitor thread got around to it, and it would - * have to double check if it had already armed the event for that connection. - * Hopefully this is nowhere close to a performance issue, but if it is, - * it can be optimized at the expense of more complex code. + * XXX: When we switch VCL we can have vbe_conn's dangling from + * XXX: the backends no longer used. When the VCL's refcount + * XXX: drops to zero we should zap them. */ #include @@ -31,6 +17,7 @@ #include #include #include +#include #include #include @@ -41,24 +28,10 @@ /* A backend IP */ -struct vbe { - unsigned magic; -#define VBE_MAGIC 0x079648f0 - unsigned ip; - TAILQ_ENTRY(vbe) list; - TAILQ_HEAD(,vbe_conn) fconn; - TAILQ_HEAD(,vbe_conn) bconn; - unsigned nconn; -}; - -static TAILQ_HEAD(,vbe) vbe_head = TAILQ_HEAD_INITIALIZER(vbe_head); +static TAILQ_HEAD(,vbe_conn) vbe_head = TAILQ_HEAD_INITIALIZER(vbe_head); static pthread_mutex_t vbemtx; -static pthread_t vbe_thread; -static struct event_base *vbe_evb; -static int vbe_pipe[2]; - /*--------------------------------------------------------------------*/ static struct vbe_conn * @@ -74,6 +47,7 @@ vbe_new_conn(void) vbc->magic = VBE_CONN_MAGIC; vbc->http = &vbc->http_mem[0]; vbc->http2 = &vbc->http_mem[1]; + vbc->fd = -1; p = (void *)(vbc + 1); http_Setup(vbc->http, p, heritage.mem_workspace); p += heritage.mem_workspace; @@ -81,15 +55,6 @@ vbe_new_conn(void) return (vbc); } -static void -vbe_delete_conn(struct vbe_conn *vb) -{ - - CHECK_OBJ_NOTNULL(vb, VBE_CONN_MAGIC); - VSL_stats->n_vbe_conn--; - free(vb); -} - /*--------------------------------------------------------------------*/ static void @@ -197,92 +162,11 @@ vbe_connect(struct backend *bp) TCP_myname(s, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1); TCP_name(ai->ai_addr, ai->ai_addrlen, abuf2, sizeof abuf2, pbuf2, sizeof pbuf2); - VSL(SLT_BackendOpen, s, "%s %s %s %s", abuf1, pbuf1, abuf2, pbuf2); + VSL(SLT_BackendOpen, s, "%s %s %s %s %s", + bp->vcl_name, abuf1, pbuf1, abuf2, pbuf2); return (s); } -/*-------------------------------------------------------------------- - * When backend connections have been used, they are passed to us through - * the vbe_pipe. If fd == -1 it has been closed and will be reclaimed, - * otherwise arm an event to monitor if the backend closes and recycle. - */ - -static void -vbe_rdp(int fd, short event, void *arg) -{ - struct vbe_conn *vc; - int i; - - (void)event; - (void)arg; - i = read(fd, &vc, sizeof vc); - assert(i == sizeof vc); - AZ(pthread_mutex_lock(&vbemtx)); - TAILQ_REMOVE(&vc->vbe->bconn, vc, list); - if (vc->fd < 0) { - vc->vbe->nconn--; - vbe_delete_conn(vc); - } else { - vc->inuse = 0; - AZ(event_add(&vc->ev, NULL)); - TAILQ_INSERT_HEAD(&vc->vbe->fconn, vc, list); - } - AZ(pthread_mutex_unlock(&vbemtx)); -} - -/*-------------------------------------------------------------------- - * A backend connection became ready. This can happen if it was reused - * in which case we unarm the event and get out of the way, or if the - * backend closed the connection in which case we clean up. - */ - -static void -vbe_rdf(int fd, short event, void *arg) -{ - struct vbe_conn *vc; - int j; - - (void)fd; - (void)event; - vc = arg; - AZ(pthread_mutex_lock(&vbemtx)); - if (vc->inuse) { - AZ(event_del(&vc->ev)); - AZ(pthread_mutex_unlock(&vbemtx)); - return; - } - AZ(ioctl(vc->fd, FIONREAD, &j)); - VSL(SLT_BackendClose, vc->fd, "Remote (%d chars)", j); - TAILQ_REMOVE(&vc->vbe->fconn, vc, list); - AZ(pthread_mutex_unlock(&vbemtx)); - AZ(event_del(&vc->ev)); - AZ(close(vc->fd)); - vbe_delete_conn(vc); -} - -/* Backend monitoring thread -----------------------------------------*/ - -static void * -vbe_main(void *priv) -{ - struct event pev; - - (void)priv; - vbe_evb = event_init(); - assert(vbe_evb != NULL); - - AZ(pipe(vbe_pipe)); - - memset(&pev, 0, sizeof pev); - event_set(&pev, vbe_pipe[0], EV_READ | EV_PERSIST, vbe_rdp, NULL); - AZ(event_base_set(vbe_evb, &pev)); - AZ(event_add(&pev, NULL)); - - AZ(event_base_loop(vbe_evb, 0)); - - INCOMPL(); -} - /* Get a backend connection ------------------------------------------ * * First locate the backend shadow, if necessary by creating one. @@ -293,62 +177,73 @@ vbe_main(void *priv) struct vbe_conn * VBE_GetFd(struct backend *bp, unsigned xid) { - struct vbe *vp; - struct vbe_conn *vc; + struct vbe_conn *vc, *vc2; + struct pollfd pfd; CHECK_OBJ_NOTNULL(bp, BACKEND_MAGIC); - AZ(pthread_mutex_lock(&vbemtx)); - vp = bp->vbe; - if (vp == NULL) { - TAILQ_FOREACH(vp, &vbe_head, list) - if (vp->ip == bp->ip) - break; - } - if (vp == NULL) { - vp = calloc(sizeof *vp, 1); - assert(vp != NULL); - vp->magic = VBE_MAGIC; - TAILQ_INIT(&vp->fconn); - TAILQ_INIT(&vp->bconn); - vp->ip = bp->ip; - bp->vbe = vp; - TAILQ_INSERT_TAIL(&vbe_head, vp, list); - } else - CHECK_OBJ(vp, VBE_MAGIC); - /* XXX: check nconn vs backend->maxcon */ - vc = TAILQ_FIRST(&vp->fconn); - if (vc != NULL) { - vc->inuse = 1; - TAILQ_REMOVE(&vp->fconn, vc, list); - TAILQ_INSERT_TAIL(&vp->bconn, vc, list); - AZ(pthread_mutex_unlock(&vbemtx)); - } else { - vc = vbe_new_conn(); - if (vc == NULL) { - AZ(pthread_mutex_unlock(&vbemtx)); - return (NULL); + while (1) { + /* + * Try all connections on this backend until we find one + * that works. If that fails, grab a free connection + * (if any) while we have the lock anyway. + */ + vc2 = NULL; + AZ(pthread_mutex_lock(&vbemtx)); + vc = TAILQ_FIRST(&bp->connlist); + if (vc != NULL) { + assert(vc->fd >= 0); + TAILQ_REMOVE(&bp->connlist, vc, list); + } else { + vc2 = TAILQ_FIRST(&vbe_head); + if (vc2 != NULL) + TAILQ_REMOVE(&vbe_head, vc2, list); } - vp->nconn++; - vc->vbe = vp; - vc->fd = -1; - vc->inuse = 1; - TAILQ_INSERT_TAIL(&vp->bconn, vc, list); AZ(pthread_mutex_unlock(&vbemtx)); + if (vc == NULL) + break; + + /* Test the connection for remote close before we use it */ + pfd.fd = vc->fd; + pfd.events = POLLIN; + pfd.revents = 0; + if (!poll(&pfd, 1, 0)) + break; + VBE_ClosedFd(vc); + } + + if (vc == NULL) { + if (vc2 == NULL) + vc = vbe_new_conn(); + else + vc = vc2; + assert(vc != NULL); + assert(vc->fd == -1); + assert(vc->backend == NULL); + } + + /* If not connected yet, do so */ + if (vc->fd < 0) { + assert(vc->backend == NULL); vc->fd = vbe_connect(bp); + AZ(pthread_mutex_lock(&vbemtx)); if (vc->fd < 0) { - AZ(pthread_mutex_lock(&vbemtx)); - TAILQ_REMOVE(&vc->vbe->bconn, vc, list); - vp->nconn--; - AZ(pthread_mutex_unlock(&vbemtx)); - vbe_delete_conn(vc); - return (NULL); + vc->backend = NULL; + TAILQ_INSERT_HEAD(&vbe_head, vc, list); + vc = NULL; + } else { + vc->backend = bp; } + AZ(pthread_mutex_unlock(&vbemtx)); + } else { + assert(vc->fd >= 0); + assert(vc->backend == bp); + } + if (vc != NULL ) { + assert(vc->fd >= 0); VSL_stats->backend_conn++; - event_set(&vc->ev, vc->fd, - EV_READ | EV_PERSIST, vbe_rdf, vc); - AZ(event_base_set(vbe_evb, &vc->ev)); + VSL(SLT_BackendXID, vc->fd, "%u", xid); + assert(vc->backend != NULL); } - VSL(SLT_BackendXID, vc->fd, "%u", xid); return (vc); } @@ -357,14 +252,17 @@ VBE_GetFd(struct backend *bp, unsigned xid) void VBE_ClosedFd(struct vbe_conn *vc) { - int i; + CHECK_OBJ_NOTNULL(vc, VBE_CONN_MAGIC); assert(vc->fd >= 0); - VSL(SLT_BackendClose, vc->fd, ""); + assert(vc->backend != NULL); + VSL(SLT_BackendClose, vc->fd, "%s", vc->backend->vcl_name); AZ(close(vc->fd)); vc->fd = -1; - i = write(vbe_pipe[1], &vc, sizeof vc); - assert(i == sizeof vc); + vc->backend = NULL; + AZ(pthread_mutex_lock(&vbemtx)); + TAILQ_INSERT_HEAD(&vbe_head, vc, list); + AZ(pthread_mutex_unlock(&vbemtx)); } /* Recycle a connection ----------------------------------------------*/ @@ -372,12 +270,15 @@ VBE_ClosedFd(struct vbe_conn *vc) void VBE_RecycleFd(struct vbe_conn *vc) { - int i; + CHECK_OBJ_NOTNULL(vc, VBE_CONN_MAGIC); + assert(vc->fd >= 0); + assert(vc->backend != NULL); VSL_stats->backend_recycle++; - VSL(SLT_BackendReuse, vc->fd, ""); - i = write(vbe_pipe[1], &vc, sizeof vc); - assert(i == sizeof vc); + VSL(SLT_BackendReuse, vc->fd, "%s", vc->backend->vcl_name); + AZ(pthread_mutex_lock(&vbemtx)); + TAILQ_INSERT_HEAD(&vc->backend->connlist, vc, list); + AZ(pthread_mutex_unlock(&vbemtx)); } /*--------------------------------------------------------------------*/ @@ -387,5 +288,4 @@ VBE_Init(void) { AZ(pthread_mutex_init(&vbemtx, NULL)); - AZ(pthread_create(&vbe_thread, NULL, vbe_main, NULL)); } diff --git a/varnish-cache/bin/varnishd/cache_fetch.c b/varnish-cache/bin/varnishd/cache_fetch.c index c5c17032..9a780251 100644 --- a/varnish-cache/bin/varnishd/cache_fetch.c +++ b/varnish-cache/bin/varnishd/cache_fetch.c @@ -219,6 +219,7 @@ FetchBody(struct sess *sp) assert(sp->obj->busy != 0); vc = sp->vbc; + sp->vbc = NULL; if (http_GetHdr(vc->http, H_Last_Modified, &b)) sp->obj->last_modified = TIM_parse(b); @@ -303,6 +304,7 @@ FetchHeaders(struct sess *sp) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); + assert(sp->vbc == NULL); sp->vbc = vc; sp->obj->entered = time(NULL); diff --git a/varnish-cache/bin/varnishd/cache_main.c b/varnish-cache/bin/varnishd/cache_main.c index f95f2aae..6672c9f5 100644 --- a/varnish-cache/bin/varnishd/cache_main.c +++ b/varnish-cache/bin/varnishd/cache_main.c @@ -14,6 +14,7 @@ #include "heritage.h" #include "shmlog.h" #include "cache.h" +#include "event.h" #include "cli_event.h" static struct event ev_keepalive; diff --git a/varnish-cache/bin/varnishd/cache_vrt.c b/varnish-cache/bin/varnishd/cache_vrt.c index 0600a4db..797407b1 100644 --- a/varnish-cache/bin/varnishd/cache_vrt.c +++ b/varnish-cache/bin/varnishd/cache_vrt.c @@ -83,6 +83,7 @@ VRT_alloc_backends(struct VCL_conf *cp) cp->backend[i] = calloc(sizeof *cp->backend[i], 1); assert(cp->backend[i] != NULL); cp->backend[i]->magic = BACKEND_MAGIC; + TAILQ_INIT(&cp->backend[i]->connlist); } }