Implement runtime part of VCL controlled hashing.

The vcl_hash() is now used to control which fields go
into the hash algorithm, and the default is stil,
as previously, the URL + Host: header.

But now it is controlled by the vcl code, with the
default vcl_hash() being:

	sub vcl_hash {
		req.hash += req.url;
		req.hash += req.http.host;
		hash;
	}

Once I get a bit further, this will be changed to

	sub vcl_hash {
		req.hash += req.url;
		if (req.http.host) {
			req.hash += req.http.host;
		} else {
			req.hash += server.ip;
		}
		hash;
	}

So that we correctly hash HTTP requests without Host:
headers, that go to a machine with multiple IP numbers.

If you want to add fields to the hash, just write
a vcl_hash that does not end in "hash;":

	sub vcl_hash {
		req.hash += req.http.cookie;
	}

If you want to override the default vcl_hash, just
say so:

	sub vcl_hash {
		req.hash += req.url;
		hash;	// do not continue into default vcl_hash
	}

git-svn-id: svn+ssh://projects.linpro.no/svn/varnish/trunk@1398 d4fa192b-c00b-0410-8231-f00ffab90ce4

diff --git a/varnish-cache/bin/varnishd/cache.h b/varnish-cache/bin/varnishd/cache.h
index 9d13c1d47be2db4709bc949adc01363f44fb5d4b..a26a6234906a044171c8c2532fa3aa4c13d9800d 100644 (file)
--- a/varnish-cache/bin/varnishd/cache.h
+++ b/varnish-cache/bin/varnishd/cache.h
@@ -291,6 +291,9 @@ struct sess {
 
        struct workreq          workreq;
        struct acct             acct;
+
+       char                    *hash_b;        /* Start of hash string */
+       char                    *hash_e;        /* End of hash string */
 };
 
 struct backend {

diff --git a/varnish-cache/bin/varnishd/cache_center.c b/varnish-cache/bin/varnishd/cache_center.c
index 19c057eb150ebe310b35aabcb37cbb703fa5068f..75e077018dffbe1fae7087081d64103e3e0ab2f8 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_center.c
+++ b/varnish-cache/bin/varnishd/cache_center.c
@@ -437,9 +437,18 @@ cnt_lookup(struct sess *sp)
 {
        struct object *o;
 
-       VCL_hash_method(sp);            /* XXX: no-op for now */
+       assert(sp->http->f > sp->http->s);
+       assert(sp->http->f >= sp->http->t);
+       if (sp->obj == NULL) {
+               sp->hash_b = sp->http->f;
+               sp->hash_e = sp->hash_b;
+               VCL_hash_method(sp);            /* XXX: no-op for now */
+
+               /* XXX check error */
+       }
 
        o = HSH_Lookup(sp);
+
        if (o == NULL) {
                /*
                 * We hit a busy object, disembark worker thread and expect
@@ -451,6 +460,14 @@ cnt_lookup(struct sess *sp)
                return (1);
        }
 
+       xxxassert (sp->hash_e == sp->http->f);
+       if (sp->hash_e == sp->http->f) {
+               /* Nobody alloc'ed after us, free again */
+               sp->http->f = sp->hash_b;
+       }
+
+       sp->hash_b = sp->hash_e = NULL;
+
        sp->obj = o;
 
        /* If we inserted a new object it's a miss */

diff --git a/varnish-cache/bin/varnishd/cache_hash.c b/varnish-cache/bin/varnishd/cache_hash.c
index b2bd45cc45cc407e84291cd6d5a92d69c52cb83a..57a354728d9143f904f8a963fc719ee6a5e2fc8c 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_hash.c
+++ b/varnish-cache/bin/varnishd/cache_hash.c
@@ -115,7 +115,6 @@ HSH_Lookup(struct sess *sp)
        struct http *h;
        struct objhead *oh;
        struct object *o;
-       char *url, *host;
 
        CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
        CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
@@ -125,9 +124,6 @@ HSH_Lookup(struct sess *sp)
        h = sp->http;
 
        HSH_Prealloc(sp);
-       url = h->hd[HTTP_HDR_URL].b;
-       if (!http_GetHdr(h, H_Host, &host))
-               host = url;
        if (sp->obj != NULL) {
                CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
                o = sp->obj;
@@ -136,7 +132,9 @@ HSH_Lookup(struct sess *sp)
                LOCK(&oh->mtx);
                goto were_back;
        }
-       oh = hash->lookup(url, host, w->nobjhead);
+VSLR(SLT_Debug, sp->fd, sp->hash_b, sp->hash_e);
+
+       oh = hash->lookup(sp->hash_b, sp->hash_e, w->nobjhead);
        CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
        if (oh == w->nobjhead)
                w->nobjhead = NULL;
@@ -157,7 +155,7 @@ HSH_Lookup(struct sess *sp)
                        /* Object banned but not reaped yet */
                } else if (o->ttl <= sp->t_req.tv_sec) {
                        /* Object expired */
-               } else if (BAN_CheckObject(o, url)) {
+               } else if (BAN_CheckObject(o, h->hd[HTTP_HDR_URL].b)) {
                        o->ttl = 0;
                        VSL(SLT_ExpBan, 0, "%u was banned", o->xid);
                        EXP_TTLchange(o);

diff --git a/varnish-cache/bin/varnishd/cache_http.c b/varnish-cache/bin/varnishd/cache_http.c
index f55145bdc2730b9f2b47b1e16335113bc56a0993..9241ee2921cf733c854b794f969d7e46534e5551 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_http.c
+++ b/varnish-cache/bin/varnishd/cache_http.c
@@ -540,6 +540,7 @@ http_header_complete(struct http *hp)
        hp->t = p;
        assert(hp->t > hp->s);
        assert(hp->t <= hp->v);
+       hp->f = hp->v;
        return (1);
 }
 

diff --git a/varnish-cache/bin/varnishd/cache_vrt.c b/varnish-cache/bin/varnishd/cache_vrt.c
index 52bc308b8b6a800d499df56d9dd0c4a49f366c28..cfa6bb54b251188f2c0759bad277d59cccdbe1bf 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_vrt.c
+++ b/varnish-cache/bin/varnishd/cache_vrt.c
@@ -273,3 +273,21 @@ VRT_r_server_ip(struct sess *sp)
 
        return ((struct sockaddr*)sp->mysockaddr);
 }
+
+/*--------------------------------------------------------------------*/
+
+void
+VRT_l_req_hash(struct sess *sp, const char *str)
+{
+       int l;
+
+       if (str == NULL)
+               str = "";
+       l = strlen(str);
+       xxxassert (sp->hash_e == sp->http->f);
+       xxxassert (sp->hash_e + l + 1 <= sp->http->e);
+       memcpy(sp->hash_e, str, l);
+       sp->hash_e[l] = '#';
+       sp->hash_e += l + 1;
+       sp->http->f += l + 1;
+}

diff --git a/varnish-cache/bin/varnishd/hash_classic.c b/varnish-cache/bin/varnishd/hash_classic.c
index 72e607200b50172b6d39ed76b53fbcf4eb40d3d3..dbafd49a34390563b48af405907a9fc36ecfaf82 100644 (file)
--- a/varnish-cache/bin/varnishd/hash_classic.c
+++ b/varnish-cache/bin/varnishd/hash_classic.c
@@ -121,22 +121,20 @@ hcl_start(void)
  */
 
 static struct objhead *
-hcl_lookup(const char *key1, const char *key2, struct objhead *noh)
+hcl_lookup(const char *b, const char *e, struct objhead *noh)
 {
        struct hcl_entry *he, *he2;
        struct hcl_hd *hp;
-       unsigned u1, digest, kl1, kl2, kl, r;
+       unsigned u1, digest, kl, r;
        int i;
 
        CHECK_OBJ_NOTNULL(noh, OBJHEAD_MAGIC);
 
-       digest = crc32_2s(key1, key2);
+       digest = crc32_l(b, e - b);
 
        u1 = digest % hcl_nhash;
        hp = &hcl_head[u1];
-       kl1 = strlen(key1) + 1;         /* Incl '/0' */
-       kl2 = strlen(key2);
-       kl = kl1 + kl2;
+       kl = e - b;
        he2 = NULL;
 
        for (r = 0; r < 2; r++ ) {
@@ -151,9 +149,7 @@ hcl_lookup(const char *key1, const char *key2, struct objhead *noh)
                                continue;
                        if (he->digest > digest)
                                break;
-                       if (memcmp(he->key, key1, kl1))
-                               continue;
-                       if (memcmp(he->key + kl1, key2, kl2))
+                       if (memcmp(he->key, b, kl))
                                continue;
                        he->refcnt++;
                        noh = he->oh;
@@ -190,8 +186,7 @@ hcl_lookup(const char *key1, const char *key2, struct objhead *noh)
                noh->hashpriv = he2;
 
                he2->key = (void*)(he2 + 1);
-               memcpy(he2->key, key1, kl1);
-               memcpy(he2->key + kl1, key2, kl2);
+               memcpy(he2->key, b, kl);
        }
        assert(he2 == NULL);            /* FlexeLint */
        INCOMPL();

diff --git a/varnish-cache/bin/varnishd/hash_simple_list.c b/varnish-cache/bin/varnishd/hash_simple_list.c
index 9d294fa05972e9b9b106bf44c381b3b8a447d58a..ca284b300ac5541a2c449511ad867f320aba7eac 100644 (file)
--- a/varnish-cache/bin/varnishd/hash_simple_list.c
+++ b/varnish-cache/bin/varnishd/hash_simple_list.c
@@ -44,8 +44,8 @@
 
 struct hsl_entry {
        TAILQ_ENTRY(hsl_entry)  list;
-       char                    *key1;
-       char                    *key2;
+       char                    *key;
+       int                     keylen;
        struct objhead          *obj;
        unsigned                refcnt;
 };
@@ -73,19 +73,19 @@ hsl_start(void)
  */
 
 static struct objhead *
-hsl_lookup(const char *key1, const char *key2, struct objhead *nobj)
+hsl_lookup(const char *b, const char *e, struct objhead *nobj)
 {
        struct hsl_entry *he, *he2;
-       int i;
+       int i, l;
 
+       l = e - b;
        LOCK(&hsl_mutex);
        TAILQ_FOREACH(he, &hsl_head, list) {
-               i = strcmp(key1, he->key1);
-               if (i < 0)
+               if (l > he->keylen)
                        continue;
-               if (i > 0)
-                       break;
-               i = strcmp(key2, he->key2);
+               if (l < he->keylen)
+                       break;;
+               i = memcmp(b, he->key, l);
                if (i < 0)
                        continue;
                if (i > 0)
@@ -100,14 +100,13 @@ hsl_lookup(const char *key1, const char *key2, struct objhead *nobj)
                UNLOCK(&hsl_mutex);
                return (NULL);
        }
-       he2 = calloc(sizeof *he2, 1);
+       he2 = calloc(sizeof *he2 + l, 1);
        XXXAN(he2);
        he2->obj = nobj;
        he2->refcnt = 1;
-       he2->key1 = strdup(key1);
-       XXXAN(he2->key1);
-       he2->key2 = strdup(key2);
-       XXXAN(he2->key2);
+       he2->key = (void*)(he2 + 1);
+       he2->keylen = l;
+       memcpy(he2->key, b, l);
        nobj->hashpriv = he2;
        if (he != NULL)
                TAILQ_INSERT_BEFORE(he, he2, list);
@@ -131,8 +130,6 @@ hsl_deref(struct objhead *obj)
        he = obj->hashpriv;
        LOCK(&hsl_mutex);
        if (--he->refcnt == 0) {
-               free(he->key1);
-               free(he->key2);
                TAILQ_REMOVE(&hsl_head, he, list);
                free(he);
                ret = 0;

diff --git a/varnish-cache/bin/varnishd/mgt_vcc.c b/varnish-cache/bin/varnishd/mgt_vcc.c
index d5fc867819dad7d103d2ae5b3113691de9e3dfd0..18681a2ed18b6413811b46fce661541d74d4cf18 100644 (file)
--- a/varnish-cache/bin/varnishd/mgt_vcc.c
+++ b/varnish-cache/bin/varnishd/mgt_vcc.c
@@ -88,6 +88,16 @@ static const char *default_vcl =
     "}\n"
     "\n"
     "sub vcl_hash {\n"
+    "    set req.hash += req.url;\n"
+#if 1
+    "    set req.hash += req.http.host;\n"
+#else
+    "    if (req.http.host) {\n"
+    "        set req.hash += req.http.host;\n"
+    "    } else {\n"
+    "        set req.hash += server.ip;\n"
+    "    }\n"
+#endif
     "    hash;\n"
     "}\n"
     "\n"