From 8323f408b0d1a9b2d7ac68e6195ecdd669cee0f4 Mon Sep 17 00:00:00 2001 From: phk Date: Wed, 19 Apr 2006 06:34:10 +0000 Subject: [PATCH] Implement enough of FetchSession and DeliverSession that we can actually deliver a cached object. git-svn-id: svn+ssh://projects.linpro.no/svn/varnish/trunk@142 d4fa192b-c00b-0410-8231-f00ffab90ce4 --- varnish-cache/bin/varnishd/Makefile.am | 1 + varnish-cache/bin/varnishd/cache.h | 6 +- varnish-cache/bin/varnishd/cache_fetch.c | 125 ++++++++++++++++++ varnish-cache/bin/varnishd/cache_pool.c | 43 +++++- varnish-cache/bin/varnishd/hash_simple_list.c | 11 ++ varnish-cache/bin/varnishlog/varnishlog.c | 15 ++- varnish-cache/include/shmlog_tags.h | 1 + varnish-cache/include/vcl_lang.h | 1 + varnish-cache/lib/libvcl/vcl_fixed_token.c | 1 + 9 files changed, 192 insertions(+), 12 deletions(-) create mode 100644 varnish-cache/bin/varnishd/cache_fetch.c diff --git a/varnish-cache/bin/varnishd/Makefile.am b/varnish-cache/bin/varnishd/Makefile.am index f8606e92..62bd8de6 100644 --- a/varnish-cache/bin/varnishd/Makefile.am +++ b/varnish-cache/bin/varnishd/Makefile.am @@ -7,6 +7,7 @@ bin_PROGRAMS = varnishd varnishd_SOURCES = \ cache_acceptor.c \ cache_backend.c \ + cache_fetch.c \ cache_httpd.c \ cache_main.c \ cache_pool.c \ diff --git a/varnish-cache/bin/varnishd/cache.h b/varnish-cache/bin/varnishd/cache.h index 1cd7753d..fff7791c 100644 --- a/varnish-cache/bin/varnishd/cache.h +++ b/varnish-cache/bin/varnishd/cache.h @@ -20,12 +20,14 @@ struct worker; typedef void hash_init_f(void); typedef struct object *hash_lookup_f(unsigned char *key, struct object *nobj); +typedef void hash_deref_f(struct object *obj); typedef void hash_purge_f(struct object *obj); struct hash_slinger { const char *name; hash_init_f *init; hash_lookup_f *lookup; + hash_deref_f *deref; hash_purge_f *purge; }; @@ -37,7 +39,7 @@ extern struct hash_slinger *hash; struct storage { TAILQ_ENTRY(storage) list; - void *ptr; + unsigned char *ptr; unsigned len; void *priv; }; @@ -72,6 +74,8 @@ void VBE_Pass(struct sess *sp); void VBE_ClosedFd(void *ptr); void VBE_RecycleFd(void *ptr); +/* cache_fetch.c */ +int FetchSession(struct worker *w, struct sess *sp); /* cache_httpd.c */ void HttpdAnalyze(struct sess *sp, int rr); diff --git a/varnish-cache/bin/varnishd/cache_fetch.c b/varnish-cache/bin/varnishd/cache_fetch.c new file mode 100644 index 00000000..44afd29a --- /dev/null +++ b/varnish-cache/bin/varnishd/cache_fetch.c @@ -0,0 +1,125 @@ +/* + * $Id$ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "libvarnish.h" +#include "shmlog.h" +#include "vcl_lang.h" +#include "cache.h" + +static void +FetchReturn(struct sess *sp) +{ + + /* do nothing */ +} + +/*--------------------------------------------------------------------*/ +int +FetchSession(struct worker *w, struct sess *sp) +{ + int fd, i; + void *fd_token; + struct sess sp2; + off_t cl; + struct storage *st; + unsigned char *p; + + fd = VBE_GetFd(sp->backend, &fd_token); + assert(fd != -1); + VSL(SLT_HandlingFetch, sp->fd, "%d", fd); + + HttpdBuildSbuf(0, 1, w->sb, sp); + i = write(fd, sbuf_data(w->sb), sbuf_len(w->sb)); + assert(i == sbuf_len(w->sb)); + + /* XXX: copy any contents */ + + memset(&sp2, 0, sizeof sp2); + sp2.rd_e = &w->e1; + sp2.fd = fd; + /* + * XXX: It might be cheaper to avoid the event_engine and simply + * XXX: read(2) the header + */ + HttpdGetHead(&sp2, w->eb, FetchReturn); + event_base_loop(w->eb, 0); + HttpdAnalyze(&sp2, 2); + + /* XXX: fill in object from headers */ + sp->obj->valid = 1; + sp->obj->cacheable = 1; + + /* XXX: unbusy, and kick other sessions into action */ + sp->obj->busy = 0; + + assert (sp2.http.H_Content_Length != NULL); /* XXX */ + + cl = strtoumax(sp2.http.H_Content_Length, NULL, 0); + + sp->handling = HND_Unclass; + sp->vcl->fetch_func(sp); + + st = stevedore->alloc(cl); + TAILQ_INSERT_TAIL(&sp->obj->store, st, list); + st->len = cl; + sp->obj->len = cl; + p = st->ptr; + + i = fcntl(sp2.fd, F_GETFL); /* XXX ? */ + i &= ~O_NONBLOCK; + i = fcntl(sp2.fd, F_SETFL, i); + + i = sp2.rcv_len - sp2.rcv_ptr; + if (i > 0) { + memcpy(p, sp2.rcv + sp2.rcv_ptr, i); + p += i; + cl -= i; + } + if (cl != 0) { + i = read(sp2.fd, p, cl); + VSL(SLT_Debug, 0, "R i %d cl %jd", i, cl); + assert(i == cl); + } + + HttpdBuildSbuf(1, 1, w->sb, &sp2); + i = write(sp->fd, sbuf_data(w->sb), sbuf_len(w->sb)); + assert(i == sbuf_len(w->sb)); + + i = write(sp->fd, st->ptr, st->len); + VSL(SLT_Debug, 0, "W i %d st->len %u", i, st->len); + assert(i == st->len); + + hash->deref(sp->obj); + + if (sp2.http.H_Connection != NULL && + !strcmp(sp2.http.H_Connection, "close")) { + close(fd); + VBE_ClosedFd(fd_token); + } else { + VBE_RecycleFd(fd_token); + } + + /* XXX: this really belongs in the acceptor */ + if (sp->rcv_len > sp->rcv_ptr) + memmove(sp->rcv, sp->rcv + sp->rcv_ptr, + sp->rcv_len - sp->rcv_ptr); + sp->rcv_len -= sp->rcv_ptr; + sp->rcv_ptr = 0; + return (1); +} diff --git a/varnish-cache/bin/varnishd/cache_pool.c b/varnish-cache/bin/varnishd/cache_pool.c index 23b3e605..b8abaf28 100644 --- a/varnish-cache/bin/varnishd/cache_pool.c +++ b/varnish-cache/bin/varnishd/cache_pool.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -40,8 +41,12 @@ LookupSession(struct worker *w, struct sess *sp) MD5Update(&ctx, sp->http.url, strlen(sp->http.url)); MD5Final(key, &ctx); o = hash->lookup(key, w->nobj); - if (o == w->nobj) + if (o == w->nobj) { + VSL(SLT_Debug, 0, "Lookup new %p %s", o, sp->http.url); w->nobj = NULL; + } else { + VSL(SLT_Debug, 0, "Lookup found %p %s", o, sp->http.url); + } /* * XXX: if obj is busy, park session on it */ @@ -59,10 +64,26 @@ LookupSession(struct worker *w, struct sess *sp) } static int -FetchSession(struct worker *w, struct sess *sp) +DeliverSession(struct worker *w, struct sess *sp) { - - assert(w == NULL); + char buf[BUFSIZ]; + int i, j; + struct storage *st; + + sprintf(buf, + "HTTP/1.1 200 OK\r\n" + "Server: Varnish\r\n" + "Content-Length: %u\r\n" + "\r\n", sp->obj->len); + + j = strlen(buf); + i = write(sp->fd, buf, j); + assert(i == j); + TAILQ_FOREACH(st, &sp->obj->store, list) { + i = write(sp->fd, st->ptr, st->len); + assert(i == st->len); + } + return (1); } static void * @@ -112,6 +133,9 @@ CacheWorker(void *priv) case HND_Fetch: done = FetchSession(&w, sp); break; + case HND_Deliver: + done = DeliverSession(&w, sp); + break; case HND_Pipe: PipeSession(&w, sp); done = 1; @@ -121,15 +145,22 @@ CacheWorker(void *priv) done = 1; break; case HND_Unclass: - case HND_Deliver: assert(sp->handling == HND_Unclass); } } + if (sp->http.H_Connection != NULL && + !strcmp(sp->http.H_Connection, "close")) { + close(sp->fd); + sp->fd = -1; + } AZ(pthread_mutex_lock(&sessmtx)); RelVCL(sp->vcl); sp->vcl = NULL; - vca_recycle_session(sp); + if (sp->fd < 0) + vca_retire_session(sp); + else + vca_recycle_session(sp); } } diff --git a/varnish-cache/bin/varnishd/hash_simple_list.c b/varnish-cache/bin/varnishd/hash_simple_list.c index 72323da0..f73152e7 100644 --- a/varnish-cache/bin/varnishd/hash_simple_list.c +++ b/varnish-cache/bin/varnishd/hash_simple_list.c @@ -55,6 +55,7 @@ hsl_lookup(unsigned char *key, struct object *nobj) assert(he2 != NULL); he2->obj = nobj; nobj->refcnt++; + memcpy(nobj->hash, key, sizeof nobj->hash); if (he != NULL) TAILQ_INSERT_BEFORE(he, he2, list); else @@ -63,6 +64,15 @@ hsl_lookup(unsigned char *key, struct object *nobj) return (nobj); } +static void +hsl_deref(struct object *obj) +{ + + AZ(pthread_mutex_lock(&hsl_mutex)); + obj->refcnt--; + AZ(pthread_mutex_unlock(&hsl_mutex)); +} + static void hsl_purge(struct object *obj) { @@ -85,5 +95,6 @@ struct hash_slinger hsl_slinger = { "simple_list", hsl_init, hsl_lookup, + hsl_deref, hsl_purge }; diff --git a/varnish-cache/bin/varnishlog/varnishlog.c b/varnish-cache/bin/varnishlog/varnishlog.c index b3b62ad2..9103a32d 100644 --- a/varnish-cache/bin/varnishlog/varnishlog.c +++ b/varnish-cache/bin/varnishlog/varnishlog.c @@ -40,6 +40,7 @@ main(int argc, char **argv) int fd; int i; unsigned u; + unsigned startup; struct shmloghead slh; unsigned char *p; @@ -74,6 +75,7 @@ main(int argc, char **argv) for (i = 0; stagnames[i].tag != SLT_ENDMARKER; i++) tagnames[stagnames[i].tag] = stagnames[i].name; + startup = 1; while (1) { p = logstart; while (1) { @@ -82,13 +84,16 @@ main(int argc, char **argv) while (*p == SLT_ENDMARKER) { fflush(stdout); sleep(1); + startup = 0; } u = (p[2] << 8) | p[3]; - printf("%02x %02d %4d %-12s <", - p[0], p[1], u, tagnames[p[0]]); - if (p[1] > 0) - fwrite(p + 4, p[1], 1, stdout); - printf(">\n"); + if (!startup) { + printf("%02x %02d %4d %-12s <", + p[0], p[1], u, tagnames[p[0]]); + if (p[1] > 0) + fwrite(p + 4, p[1], 1, stdout); + printf(">\n"); + } p += p[1] + 4; } } diff --git a/varnish-cache/include/shmlog_tags.h b/varnish-cache/include/shmlog_tags.h index 2456ccd8..be79369b 100644 --- a/varnish-cache/include/shmlog_tags.h +++ b/varnish-cache/include/shmlog_tags.h @@ -12,6 +12,7 @@ SLTM(SessionOpen) SLTM(SessionReuse) SLTM(SessionClose) SLTM(ClientAddr) +SLTM(HandlingFetch) SLTM(HandlingPass) SLTM(HandlingPipe) SLTM(Request) diff --git a/varnish-cache/include/vcl_lang.h b/varnish-cache/include/vcl_lang.h index 8afd4be0..1eb91d04 100644 --- a/varnish-cache/include/vcl_lang.h +++ b/varnish-cache/include/vcl_lang.h @@ -47,6 +47,7 @@ struct object { unsigned cacheable; unsigned busy; + unsigned len; TAILQ_HEAD(, storage) store; }; diff --git a/varnish-cache/lib/libvcl/vcl_fixed_token.c b/varnish-cache/lib/libvcl/vcl_fixed_token.c index 1b5becca..5030bd4a 100644 --- a/varnish-cache/lib/libvcl/vcl_fixed_token.c +++ b/varnish-cache/lib/libvcl/vcl_fixed_token.c @@ -485,6 +485,7 @@ vcl_output_lang_h(FILE *f) fputs(" unsigned cacheable;\n", f); fputs("\n", f); fputs(" unsigned busy;\n", f); + fputs(" unsigned len;\n", f); fputs("\n", f); fputs(" TAILQ_HEAD(, storage) store;\n", f); fputs("};\n", f); -- 2.39.5