From: phk Date: Wed, 9 May 2007 14:28:50 +0000 (+0000) Subject: Implement runtime part of VCL controlled hashing. X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1edee51c8a57f1bf8f2a9122656923318df440b2;p=varnish Implement runtime part of VCL controlled hashing. The vcl_hash() is now used to control which fields go into the hash algorithm, and the default is stil, as previously, the URL + Host: header. But now it is controlled by the vcl code, with the default vcl_hash() being: sub vcl_hash { req.hash += req.url; req.hash += req.http.host; hash; } Once I get a bit further, this will be changed to sub vcl_hash { req.hash += req.url; if (req.http.host) { req.hash += req.http.host; } else { req.hash += server.ip; } hash; } So that we correctly hash HTTP requests without Host: headers, that go to a machine with multiple IP numbers. If you want to add fields to the hash, just write a vcl_hash that does not end in "hash;": sub vcl_hash { req.hash += req.http.cookie; } If you want to override the default vcl_hash, just say so: sub vcl_hash { req.hash += req.url; hash; // do not continue into default vcl_hash } git-svn-id: svn+ssh://projects.linpro.no/svn/varnish/trunk@1398 d4fa192b-c00b-0410-8231-f00ffab90ce4 --- diff --git a/varnish-cache/bin/varnishd/cache.h b/varnish-cache/bin/varnishd/cache.h index 9d13c1d4..a26a6234 100644 --- a/varnish-cache/bin/varnishd/cache.h +++ b/varnish-cache/bin/varnishd/cache.h @@ -291,6 +291,9 @@ struct sess { struct workreq workreq; struct acct acct; + + char *hash_b; /* Start of hash string */ + char *hash_e; /* End of hash string */ }; struct backend { diff --git a/varnish-cache/bin/varnishd/cache_center.c b/varnish-cache/bin/varnishd/cache_center.c index 19c057eb..75e07701 100644 --- a/varnish-cache/bin/varnishd/cache_center.c +++ b/varnish-cache/bin/varnishd/cache_center.c @@ -437,9 +437,18 @@ cnt_lookup(struct sess *sp) { struct object *o; - VCL_hash_method(sp); /* XXX: no-op for now */ + assert(sp->http->f > sp->http->s); + assert(sp->http->f >= sp->http->t); + if (sp->obj == NULL) { + sp->hash_b = sp->http->f; + sp->hash_e = sp->hash_b; + VCL_hash_method(sp); /* XXX: no-op for now */ + + /* XXX check error */ + } o = HSH_Lookup(sp); + if (o == NULL) { /* * We hit a busy object, disembark worker thread and expect @@ -451,6 +460,14 @@ cnt_lookup(struct sess *sp) return (1); } + xxxassert (sp->hash_e == sp->http->f); + if (sp->hash_e == sp->http->f) { + /* Nobody alloc'ed after us, free again */ + sp->http->f = sp->hash_b; + } + + sp->hash_b = sp->hash_e = NULL; + sp->obj = o; /* If we inserted a new object it's a miss */ diff --git a/varnish-cache/bin/varnishd/cache_hash.c b/varnish-cache/bin/varnishd/cache_hash.c index b2bd45cc..57a35472 100644 --- a/varnish-cache/bin/varnishd/cache_hash.c +++ b/varnish-cache/bin/varnishd/cache_hash.c @@ -115,7 +115,6 @@ HSH_Lookup(struct sess *sp) struct http *h; struct objhead *oh; struct object *o; - char *url, *host; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); @@ -125,9 +124,6 @@ HSH_Lookup(struct sess *sp) h = sp->http; HSH_Prealloc(sp); - url = h->hd[HTTP_HDR_URL].b; - if (!http_GetHdr(h, H_Host, &host)) - host = url; if (sp->obj != NULL) { CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); o = sp->obj; @@ -136,7 +132,9 @@ HSH_Lookup(struct sess *sp) LOCK(&oh->mtx); goto were_back; } - oh = hash->lookup(url, host, w->nobjhead); +VSLR(SLT_Debug, sp->fd, sp->hash_b, sp->hash_e); + + oh = hash->lookup(sp->hash_b, sp->hash_e, w->nobjhead); CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); if (oh == w->nobjhead) w->nobjhead = NULL; @@ -157,7 +155,7 @@ HSH_Lookup(struct sess *sp) /* Object banned but not reaped yet */ } else if (o->ttl <= sp->t_req.tv_sec) { /* Object expired */ - } else if (BAN_CheckObject(o, url)) { + } else if (BAN_CheckObject(o, h->hd[HTTP_HDR_URL].b)) { o->ttl = 0; VSL(SLT_ExpBan, 0, "%u was banned", o->xid); EXP_TTLchange(o); diff --git a/varnish-cache/bin/varnishd/cache_http.c b/varnish-cache/bin/varnishd/cache_http.c index f55145bd..9241ee29 100644 --- a/varnish-cache/bin/varnishd/cache_http.c +++ b/varnish-cache/bin/varnishd/cache_http.c @@ -540,6 +540,7 @@ http_header_complete(struct http *hp) hp->t = p; assert(hp->t > hp->s); assert(hp->t <= hp->v); + hp->f = hp->v; return (1); } diff --git a/varnish-cache/bin/varnishd/cache_vrt.c b/varnish-cache/bin/varnishd/cache_vrt.c index 52bc308b..cfa6bb54 100644 --- a/varnish-cache/bin/varnishd/cache_vrt.c +++ b/varnish-cache/bin/varnishd/cache_vrt.c @@ -273,3 +273,21 @@ VRT_r_server_ip(struct sess *sp) return ((struct sockaddr*)sp->mysockaddr); } + +/*--------------------------------------------------------------------*/ + +void +VRT_l_req_hash(struct sess *sp, const char *str) +{ + int l; + + if (str == NULL) + str = ""; + l = strlen(str); + xxxassert (sp->hash_e == sp->http->f); + xxxassert (sp->hash_e + l + 1 <= sp->http->e); + memcpy(sp->hash_e, str, l); + sp->hash_e[l] = '#'; + sp->hash_e += l + 1; + sp->http->f += l + 1; +} diff --git a/varnish-cache/bin/varnishd/hash_classic.c b/varnish-cache/bin/varnishd/hash_classic.c index 72e60720..dbafd49a 100644 --- a/varnish-cache/bin/varnishd/hash_classic.c +++ b/varnish-cache/bin/varnishd/hash_classic.c @@ -121,22 +121,20 @@ hcl_start(void) */ static struct objhead * -hcl_lookup(const char *key1, const char *key2, struct objhead *noh) +hcl_lookup(const char *b, const char *e, struct objhead *noh) { struct hcl_entry *he, *he2; struct hcl_hd *hp; - unsigned u1, digest, kl1, kl2, kl, r; + unsigned u1, digest, kl, r; int i; CHECK_OBJ_NOTNULL(noh, OBJHEAD_MAGIC); - digest = crc32_2s(key1, key2); + digest = crc32_l(b, e - b); u1 = digest % hcl_nhash; hp = &hcl_head[u1]; - kl1 = strlen(key1) + 1; /* Incl '/0' */ - kl2 = strlen(key2); - kl = kl1 + kl2; + kl = e - b; he2 = NULL; for (r = 0; r < 2; r++ ) { @@ -151,9 +149,7 @@ hcl_lookup(const char *key1, const char *key2, struct objhead *noh) continue; if (he->digest > digest) break; - if (memcmp(he->key, key1, kl1)) - continue; - if (memcmp(he->key + kl1, key2, kl2)) + if (memcmp(he->key, b, kl)) continue; he->refcnt++; noh = he->oh; @@ -190,8 +186,7 @@ hcl_lookup(const char *key1, const char *key2, struct objhead *noh) noh->hashpriv = he2; he2->key = (void*)(he2 + 1); - memcpy(he2->key, key1, kl1); - memcpy(he2->key + kl1, key2, kl2); + memcpy(he2->key, b, kl); } assert(he2 == NULL); /* FlexeLint */ INCOMPL(); diff --git a/varnish-cache/bin/varnishd/hash_simple_list.c b/varnish-cache/bin/varnishd/hash_simple_list.c index 9d294fa0..ca284b30 100644 --- a/varnish-cache/bin/varnishd/hash_simple_list.c +++ b/varnish-cache/bin/varnishd/hash_simple_list.c @@ -44,8 +44,8 @@ struct hsl_entry { TAILQ_ENTRY(hsl_entry) list; - char *key1; - char *key2; + char *key; + int keylen; struct objhead *obj; unsigned refcnt; }; @@ -73,19 +73,19 @@ hsl_start(void) */ static struct objhead * -hsl_lookup(const char *key1, const char *key2, struct objhead *nobj) +hsl_lookup(const char *b, const char *e, struct objhead *nobj) { struct hsl_entry *he, *he2; - int i; + int i, l; + l = e - b; LOCK(&hsl_mutex); TAILQ_FOREACH(he, &hsl_head, list) { - i = strcmp(key1, he->key1); - if (i < 0) + if (l > he->keylen) continue; - if (i > 0) - break; - i = strcmp(key2, he->key2); + if (l < he->keylen) + break;; + i = memcmp(b, he->key, l); if (i < 0) continue; if (i > 0) @@ -100,14 +100,13 @@ hsl_lookup(const char *key1, const char *key2, struct objhead *nobj) UNLOCK(&hsl_mutex); return (NULL); } - he2 = calloc(sizeof *he2, 1); + he2 = calloc(sizeof *he2 + l, 1); XXXAN(he2); he2->obj = nobj; he2->refcnt = 1; - he2->key1 = strdup(key1); - XXXAN(he2->key1); - he2->key2 = strdup(key2); - XXXAN(he2->key2); + he2->key = (void*)(he2 + 1); + he2->keylen = l; + memcpy(he2->key, b, l); nobj->hashpriv = he2; if (he != NULL) TAILQ_INSERT_BEFORE(he, he2, list); @@ -131,8 +130,6 @@ hsl_deref(struct objhead *obj) he = obj->hashpriv; LOCK(&hsl_mutex); if (--he->refcnt == 0) { - free(he->key1); - free(he->key2); TAILQ_REMOVE(&hsl_head, he, list); free(he); ret = 0; diff --git a/varnish-cache/bin/varnishd/mgt_vcc.c b/varnish-cache/bin/varnishd/mgt_vcc.c index d5fc8678..18681a2e 100644 --- a/varnish-cache/bin/varnishd/mgt_vcc.c +++ b/varnish-cache/bin/varnishd/mgt_vcc.c @@ -88,6 +88,16 @@ static const char *default_vcl = "}\n" "\n" "sub vcl_hash {\n" + " set req.hash += req.url;\n" +#if 1 + " set req.hash += req.http.host;\n" +#else + " if (req.http.host) {\n" + " set req.hash += req.http.host;\n" + " } else {\n" + " set req.hash += server.ip;\n" + " }\n" +#endif " hash;\n" "}\n" "\n"