From: phk Date: Fri, 16 Jun 2006 10:22:40 +0000 (+0000) Subject: Initial http_GetHdrField() function. X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=3c88fcba8de6673f749dd85077412a13b53ac856;p=varnish Initial http_GetHdrField() function. Improve chunked encoding, allocate big storage chunks and trim the last one at the end, instead of one storage chunk for each chunk the remote server sends. Call RFC2616 policy code. Store headers from backend in cache and return to client. git-svn-id: svn+ssh://projects.linpro.no/svn/varnish/trunk@194 d4fa192b-c00b-0410-8231-f00ffab90ce4 --- diff --git a/varnish-cache/bin/varnishd/cache.h b/varnish-cache/bin/varnishd/cache.h index 08bd82e1..f547968e 100644 --- a/varnish-cache/bin/varnishd/cache.h +++ b/varnish-cache/bin/varnishd/cache.h @@ -41,6 +41,7 @@ struct storage { TAILQ_ENTRY(storage) list; unsigned char *ptr; unsigned len; + unsigned space; void *priv; struct stevedore *stevedore; }; @@ -81,6 +82,7 @@ struct http; struct http *http_New(void); void http_Delete(struct http *hp); int http_GetHdr(struct http *hp, const char *hdr, char **ptr); +int http_GetHdrField(struct http *hp, const char *hdr, const char *field, char **ptr); int http_GetStatus(struct http *hp); int http_HdrIs(struct http *hp, const char *hdr, const char *val); int http_GetTail(struct http *hp, unsigned len, char **b, char **e); @@ -124,3 +126,7 @@ cli_func_t cli_func_config_load; cli_func_t cli_func_config_unload; cli_func_t cli_func_config_use; #endif + +/* rfc2616.c */ +void RFC2616_Age(struct http *hp, time_t, time_t); + diff --git a/varnish-cache/bin/varnishd/cache_fetch.c b/varnish-cache/bin/varnishd/cache_fetch.c index a699e5ac..87abf185 100644 --- a/varnish-cache/bin/varnishd/cache_fetch.c +++ b/varnish-cache/bin/varnishd/cache_fetch.c @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -22,6 +23,17 @@ #include "vcl_lang.h" #include "cache.h" +/* + * Chunked encoding is a hack. We prefer to have a single or a few + * large storage objects, and a terribly long list of small ones. + * If our stevedore can trim, we alloc big chunks and trim the last one + * at the end know the result. + * + * Good testcase: http://www.washingtonpost.com/ + */ +#define CHUNK_PREALLOC (128 * 1024) + +/*--------------------------------------------------------------------*/ static int fetch_straight(struct worker *w, struct sess *sp, int fd, struct http *hp, char *b) { @@ -45,7 +57,6 @@ fetch_straight(struct worker *w, struct sess *sp, int fd, struct http *hp, char if (http_GetTail(hp, cl, &b, &e)) { i = e - b; - VSL(SLT_Debug, 0, "Fetch_Tail %jd %d", cl, i); memcpy(p, b, i); p += i; cl -= i; @@ -53,7 +64,6 @@ fetch_straight(struct worker *w, struct sess *sp, int fd, struct http *hp, char while (cl != 0) { i = read(fd, p, cl); - VSL(SLT_Debug, 0, "Fetch_Read %jd %d", cl, i); assert(i > 0); p += i; cl -= i; @@ -68,6 +78,9 @@ fetch_straight(struct worker *w, struct sess *sp, int fd, struct http *hp, char } +/*--------------------------------------------------------------------*/ +/* XXX: Cleanup. It must be possible somehow :-( */ + static int fetch_chunked(struct worker *w, struct sess *sp, int fd, struct http *hp) { @@ -75,7 +88,7 @@ fetch_chunked(struct worker *w, struct sess *sp, int fd, struct http *hp) char *b, *q, *e; unsigned char *p; struct storage *st; - unsigned u; + unsigned u, v; char buf[20]; char *bp, *be; @@ -84,46 +97,92 @@ fetch_chunked(struct worker *w, struct sess *sp, int fd, struct http *hp) i = fcntl(fd, F_SETFL, i); be = buf + sizeof buf; + bp = buf; + st = NULL; while (1) { - bp = buf; if (http_GetTail(hp, be - bp, &b, &e)) { memcpy(bp, b, e - b); bp += e - b; + *bp = '\0'; } else { i = read(fd, bp, be - bp); assert(i >= 0); bp += i; + *bp = '\0'; } u = strtoul(buf, &q, 16); - if (q == NULL || (*q != '\n' && *q != '\r')) + if (q == NULL || q == buf) continue; + assert(isspace(*q)); + while (*q == '\t' || *q == ' ') + q++; if (*q == '\r') q++; assert(*q == '\n'); q++; if (u == 0) break; - st = stevedore->alloc(stevedore, u); - TAILQ_INSERT_TAIL(&sp->obj->store, st, list); - st->len = u; sp->obj->len += u; - p = st->ptr; - memcpy(p, q, bp - q); - p += bp - q; - u -= bp - q; - if (http_GetTail(hp, u, &b, &e)) { - memcpy(p, b, e - b); - p += e - b; - u -= e - b; - } + while (u > 0) { - i = read(fd, p, u); - assert(i > 0); - u -= i; - p += i; + if (st != NULL && st->len < st->space) { + p = st->ptr + st->len; + } else { + st = stevedore->alloc(stevedore, + stevedore->trim == NULL ? u : CHUNK_PREALLOC); + TAILQ_INSERT_TAIL(&sp->obj->store, st, list); + p = st->ptr; + } + v = st->space - st->len; + if (v > u) + v = u; + + i = bp - q; + if (i == 0) { + } else if (v > i) { + memcpy(p, q, i); + p += i; + st->len += i; + u -= i; + v -= i; + q = bp = buf; + } else if (i >= v) { + memcpy(p, q, v); + p += v; + st->len += i; + q += v; + u -= v; + v -= v; + if (u == 0 && bp > q) { + memcpy(buf, q, bp - q); + q = bp = buf + (bp - q); + } + } + if (u == 0) + break; + if (v == 0) + continue; + if (http_GetTail(hp, v, &b, &e)) { + memcpy(p, b, e - b); + p += e - b; + st->len += e - b; + v -= e - b; + u -= e - b; + } + while (v > 0) { + i = read(fd, p, v); + assert(i > 0); + st->len += i; + v -= i; + u -= i; + p += i; + } } } + if (st != NULL && stevedore->trim != NULL) + stevedore->trim(st, st->len); + http_BuildSbuf(2, w->sb, hp); vca_write_obj(sp, w->sb); @@ -141,6 +200,7 @@ FetchSession(struct worker *w, struct sess *sp) void *fd_token; struct http *hp; char *b; + time_t t_req, t_resp; fd = VBE_GetFd(sp->backend, &fd_token); assert(fd != -1); @@ -150,6 +210,7 @@ FetchSession(struct worker *w, struct sess *sp) http_BuildSbuf(1, w->sb, sp->http); i = write(fd, sbuf_data(w->sb), sbuf_len(w->sb)); assert(i == sbuf_len(w->sb)); + time(&t_req); /* XXX: copy any contents */ @@ -159,15 +220,21 @@ FetchSession(struct worker *w, struct sess *sp) */ http_RecvHead(hp, fd, w->eb, NULL, NULL); event_base_loop(w->eb, 0); + time(&t_resp); http_Dissect(hp, fd, 2); + RFC2616_Age(hp, t_req, t_resp); + switch (http_GetStatus(hp)) { case 200: + case 301: + http_BuildSbuf(3, w->sb, hp); /* XXX: fill in object from headers */ sp->obj->valid = 1; sp->obj->cacheable = 1; + sp->obj->header = strdup(sbuf_data(w->sb)); break; - case 301: + case 391: sp->obj->valid = 0; sp->obj->cacheable = 0; break; diff --git a/varnish-cache/bin/varnishd/cache_http.c b/varnish-cache/bin/varnishd/cache_http.c index 9b4b01d1..35924c99 100644 --- a/varnish-cache/bin/varnishd/cache_http.c +++ b/varnish-cache/bin/varnishd/cache_http.c @@ -101,6 +101,39 @@ http_GetHdr(struct http *hp, const char *hdr, char **ptr) return (0); } +int +http_GetHdrField(struct http *hp, const char *hdr, const char *field, char **ptr) +{ + char *h; + int fl; + + if (!http_GetHdr(hp, hdr, &h)) + return (0); + fl = strlen(field); + while (*h) { + if (isspace(*h)) { + h++; + continue; + } + if (*h == ',') { + h++; + continue; + } + if (memcmp(h, field, fl) || + isalpha(h[fl]) || h[fl] == '-') { + while (*h && !(isspace(*h) || *h == ',')) + h++; + continue; + } + if (h[fl] == '=') + *ptr = &h[fl + 1]; + else + *ptr = NULL; + return (1); + } + return (0); +} + int http_HdrIs(struct http *hp, const char *hdr, const char *val) { @@ -356,7 +389,7 @@ http_BuildSbuf(int resp, struct sbuf *sb, struct http *hp) sbuf_clear(sb); assert(sb != NULL); - if (resp == 2) { + if (resp == 2 || resp == 3) { sbuf_cat(sb, hp->proto); sbuf_cat(sb, " "); sbuf_cat(sb, hp->status); @@ -382,6 +415,7 @@ http_BuildSbuf(int resp, struct sbuf *sb, struct http *hp) sbuf_cat(sb, hp->hdr[u]); sbuf_cat(sb, "\r\n"); } - sbuf_cat(sb, "\r\n"); + if (resp != 3) + sbuf_cat(sb, "\r\n"); sbuf_finish(sb); } diff --git a/varnish-cache/bin/varnishd/cache_pool.c b/varnish-cache/bin/varnishd/cache_pool.c index 597c93ec..6a08e6c4 100644 --- a/varnish-cache/bin/varnishd/cache_pool.c +++ b/varnish-cache/bin/varnishd/cache_pool.c @@ -71,10 +71,9 @@ DeliverSession(struct worker *w, struct sess *sp) sbuf_clear(w->sb); sbuf_printf(w->sb, - "HTTP/1.1 200 OK\r\n" - "Server: Varnish\r\n" + "%sServer: Varnish\r\n" "Content-Length: %u\r\n" - "\r\n", sp->obj->len); + "\r\n", sp->obj->header, sp->obj->len); vca_write_obj(sp, w->sb); return (1);