Initial http_GetHdrField() function.

Improve chunked encoding, allocate big storage chunks and trim the
last one at the end, instead of one storage chunk for each chunk
the remote server sends.

Call RFC2616 policy code.

Store headers from backend in cache and return to client.

git-svn-id: svn+ssh://projects.linpro.no/svn/varnish/trunk@194 d4fa192b-c00b-0410-8231-f00ffab90ce4

diff --git a/varnish-cache/bin/varnishd/cache.h b/varnish-cache/bin/varnishd/cache.h
index 08bd82e11b12457c929981866b46a73b721f6f69..f547968e195cf6005a6cc9b83c2588b514c131a3 100644 (file)
--- a/varnish-cache/bin/varnishd/cache.h
+++ b/varnish-cache/bin/varnishd/cache.h
@@ -41,6 +41,7 @@ struct storage {
        TAILQ_ENTRY(storage)    list;
        unsigned char           *ptr;
        unsigned                len;
+       unsigned                space;
        void                    *priv;
        struct stevedore        *stevedore;
 };
@@ -81,6 +82,7 @@ struct http;
 struct http *http_New(void);
 void http_Delete(struct http *hp);
 int http_GetHdr(struct http *hp, const char *hdr, char **ptr);
+int http_GetHdrField(struct http *hp, const char *hdr, const char *field, char **ptr);
 int http_GetStatus(struct http *hp);
 int http_HdrIs(struct http *hp, const char *hdr, const char *val);
 int http_GetTail(struct http *hp, unsigned len, char **b, char **e);
@@ -124,3 +126,7 @@ cli_func_t  cli_func_config_load;
 cli_func_t     cli_func_config_unload;
 cli_func_t     cli_func_config_use;
 #endif
+
+/* rfc2616.c */
+void RFC2616_Age(struct http *hp, time_t, time_t);
+

diff --git a/varnish-cache/bin/varnishd/cache_fetch.c b/varnish-cache/bin/varnishd/cache_fetch.c
index a699e5ac68370c0808fec4b15bae1c7919c1c90c..87abf185eb5ced5a14ca58d2c5668e9d63a5ec38 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_fetch.c
+++ b/varnish-cache/bin/varnishd/cache_fetch.c
@@ -4,6 +4,7 @@
 
 #include <assert.h>
 #include <stdio.h>
+#include <ctype.h>
 #include <inttypes.h>
 #include <unistd.h>
 #include <string.h>
@@ -22,6 +23,17 @@
 #include "vcl_lang.h"
 #include "cache.h"
 
+/*
+ * Chunked encoding is a hack.  We prefer to have a single or a few
+ * large storage objects, and a terribly long list of small ones.
+ * If our stevedore can trim, we alloc big chunks and trim the last one
+ * at the end know the result.
+ *
+ * Good testcase: http://www.washingtonpost.com/
+ */
+#define CHUNK_PREALLOC         (128 * 1024)
+
+/*--------------------------------------------------------------------*/
 static int
 fetch_straight(struct worker *w, struct sess *sp, int fd, struct http *hp, char *b)
 {
@@ -45,7 +57,6 @@ fetch_straight(struct worker *w, struct sess *sp, int fd, struct http *hp, char
 
        if (http_GetTail(hp, cl, &b, &e)) {
                i = e - b;
-               VSL(SLT_Debug, 0, "Fetch_Tail %jd %d", cl, i);
                memcpy(p, b, i);
                p += i;
                cl -= i;
@@ -53,7 +64,6 @@ fetch_straight(struct worker *w, struct sess *sp, int fd, struct http *hp, char
 
        while (cl != 0) {
                i = read(fd, p, cl);
-               VSL(SLT_Debug, 0, "Fetch_Read %jd %d", cl, i);
                assert(i > 0);
                p += i;
                cl -= i;
@@ -68,6 +78,9 @@ fetch_straight(struct worker *w, struct sess *sp, int fd, struct http *hp, char
 
 }
 
+/*--------------------------------------------------------------------*/
+/* XXX: Cleanup.  It must be possible somehow :-( */
+
 static int
 fetch_chunked(struct worker *w, struct sess *sp, int fd, struct http *hp)
 {
@@ -75,7 +88,7 @@ fetch_chunked(struct worker *w, struct sess *sp, int fd, struct http *hp)
        char *b, *q, *e;
        unsigned char *p;
        struct storage *st;
-       unsigned u;
+       unsigned u, v;
        char buf[20];
        char *bp, *be;
 
@@ -84,46 +97,92 @@ fetch_chunked(struct worker *w, struct sess *sp, int fd, struct http *hp)
        i = fcntl(fd, F_SETFL, i);
 
        be = buf + sizeof buf;
+       bp = buf;
+       st = NULL;
        while (1) {
-               bp = buf;
                if (http_GetTail(hp, be - bp, &b, &e)) {
                        memcpy(bp, b, e - b);
                        bp += e - b;
+                       *bp = '\0';
                } else {
                        i = read(fd, bp, be - bp);
                        assert(i >= 0);
                        bp += i;
+                       *bp = '\0';
                }
                u = strtoul(buf, &q, 16);
-               if (q == NULL || (*q != '\n' && *q != '\r'))
+               if (q == NULL || q == buf)
                        continue;
+               assert(isspace(*q));
+               while (*q == '\t' || *q == ' ')
+                       q++;
                if (*q == '\r')
                        q++;
                assert(*q == '\n');
                q++;
                if (u == 0)
                        break;
-               st = stevedore->alloc(stevedore, u);
-               TAILQ_INSERT_TAIL(&sp->obj->store, st, list);
-               st->len = u;
                sp->obj->len += u;
-               p = st->ptr;
-               memcpy(p, q, bp - q);
-               p += bp - q;
-               u -= bp - q;
-               if (http_GetTail(hp, u, &b, &e)) {
-                       memcpy(p, b, e - b);
-                       p += e - b;
-                       u -= e - b;
-               }
+
                while (u > 0) {
-                       i = read(fd, p, u);
-                       assert(i > 0);
-                       u -= i;
-                       p += i;
+                       if (st != NULL && st->len < st->space) {
+                               p = st->ptr + st->len;
+                       } else {
+                               st = stevedore->alloc(stevedore,
+                                   stevedore->trim == NULL ? u : CHUNK_PREALLOC);
+                               TAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+                               p = st->ptr;
+                       }
+                       v = st->space - st->len;
+                       if (v > u)
+                               v = u;
+
+                       i = bp - q;
+                       if (i == 0) {
+                       } else if (v > i) {
+                               memcpy(p, q, i);
+                               p += i;
+                               st->len += i;
+                               u -= i;
+                               v -= i;
+                               q = bp = buf;
+                       } else if (i >= v) {
+                               memcpy(p, q, v);
+                               p += v;
+                               st->len += i;
+                               q += v;
+                               u -= v;
+                               v -= v;
+                               if (u == 0 && bp > q) {
+                                       memcpy(buf, q, bp - q);
+                                       q = bp = buf + (bp - q);
+                               }
+                       }
+                       if (u == 0)
+                               break;
+                       if (v == 0)
+                               continue;
+                       if (http_GetTail(hp, v, &b, &e)) {
+                               memcpy(p, b, e - b);
+                               p += e - b;
+                               st->len += e - b;
+                               v -= e - b;
+                               u -= e - b;
+                       }
+                       while (v > 0) {
+                               i = read(fd, p, v);
+                               assert(i > 0);
+                               st->len += i;
+                               v -= i;
+                               u -= i;
+                               p += i;
+                       }
                }
        }
 
+       if (st != NULL && stevedore->trim != NULL)
+               stevedore->trim(st, st->len);
+
        http_BuildSbuf(2, w->sb, hp);
 
        vca_write_obj(sp, w->sb);
@@ -141,6 +200,7 @@ FetchSession(struct worker *w, struct sess *sp)
        void *fd_token;
        struct http *hp;
        char *b;
+       time_t t_req, t_resp;
 
        fd = VBE_GetFd(sp->backend, &fd_token);
        assert(fd != -1);
@@ -150,6 +210,7 @@ FetchSession(struct worker *w, struct sess *sp)
        http_BuildSbuf(1, w->sb, sp->http);
        i = write(fd, sbuf_data(w->sb), sbuf_len(w->sb));
        assert(i == sbuf_len(w->sb));
+       time(&t_req);
 
        /* XXX: copy any contents */
 
@@ -159,15 +220,21 @@ FetchSession(struct worker *w, struct sess *sp)
         */
        http_RecvHead(hp, fd, w->eb, NULL, NULL);
        event_base_loop(w->eb, 0);
+       time(&t_resp);
        http_Dissect(hp, fd, 2);
 
+       RFC2616_Age(hp, t_req, t_resp);
+
        switch (http_GetStatus(hp)) {
        case 200:
+       case 301:
+               http_BuildSbuf(3, w->sb, hp);
                /* XXX: fill in object from headers */
                sp->obj->valid = 1;
                sp->obj->cacheable = 1;
+               sp->obj->header = strdup(sbuf_data(w->sb));
                break;
-       case 301:
+       case 391:
                sp->obj->valid = 0;
                sp->obj->cacheable = 0;
                break;

diff --git a/varnish-cache/bin/varnishd/cache_http.c b/varnish-cache/bin/varnishd/cache_http.c
index 9b4b01d1ef00d63bced1e067d924843f5cb8491b..35924c99ac41defa70465a6027b29380d98b87ef 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_http.c
+++ b/varnish-cache/bin/varnishd/cache_http.c
@@ -101,6 +101,39 @@ http_GetHdr(struct http *hp, const char *hdr, char **ptr)
        return (0);
 }
 
+int
+http_GetHdrField(struct http *hp, const char *hdr, const char *field, char **ptr)
+{
+       char *h;
+       int fl;
+
+       if (!http_GetHdr(hp, hdr, &h))
+               return (0);
+       fl = strlen(field);
+       while (*h) {
+               if (isspace(*h)) {
+                       h++;
+                       continue;
+               }
+               if (*h == ',') {
+                       h++;
+                       continue;
+               }
+               if (memcmp(h, field, fl) ||
+                   isalpha(h[fl]) || h[fl] == '-') {
+                       while (*h && !(isspace(*h) || *h == ','))
+                               h++;
+                       continue;
+               }
+               if (h[fl] == '=')
+                       *ptr = &h[fl + 1];
+               else
+                       *ptr = NULL;
+               return (1);
+       }
+       return (0);
+}
+
 int
 http_HdrIs(struct http *hp, const char *hdr, const char *val)
 {
@@ -356,7 +389,7 @@ http_BuildSbuf(int resp, struct sbuf *sb, struct http *hp)
 
        sbuf_clear(sb);
        assert(sb != NULL);
-       if (resp == 2) {
+       if (resp == 2 || resp == 3) {
                sbuf_cat(sb, hp->proto);
                sbuf_cat(sb, " ");
                sbuf_cat(sb, hp->status);
@@ -382,6 +415,7 @@ http_BuildSbuf(int resp, struct sbuf *sb, struct http *hp)
                sbuf_cat(sb, hp->hdr[u]);
                sbuf_cat(sb, "\r\n");
        }
-       sbuf_cat(sb, "\r\n");
+       if (resp != 3)
+               sbuf_cat(sb, "\r\n");
        sbuf_finish(sb);
 }

diff --git a/varnish-cache/bin/varnishd/cache_pool.c b/varnish-cache/bin/varnishd/cache_pool.c
index 597c93ec882a1bd403a9dac9330bcf22fd1c05fc..6a08e6c42f26260585d1a460a3bf3259b397baee 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_pool.c
+++ b/varnish-cache/bin/varnishd/cache_pool.c
@@ -71,10 +71,9 @@ DeliverSession(struct worker *w, struct sess *sp)
 
        sbuf_clear(w->sb);
        sbuf_printf(w->sb,
-           "HTTP/1.1 200 OK\r\n"
-           "Server: Varnish\r\n"
+           "%sServer: Varnish\r\n"
            "Content-Length: %u\r\n"
-           "\r\n", sp->obj->len);
+           "\r\n", sp->obj->header, sp->obj->len);
 
        vca_write_obj(sp, w->sb);
        return (1);