From: phk Date: Wed, 3 Dec 2008 10:49:34 +0000 (+0000) Subject: Add preliminary version of lock-less tree based lookup (see below) X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=433763e070688143a1a308990460c1838cad8d34;p=varnish Add preliminary version of lock-less tree based lookup (see below) Enable SHA256 digests by default, and put it in the objhead. This increases the size of the objhead by 32 bytes, but may drop a bit again later, when other now unnecessary fields go away. Test SHA256 for correct operation on startup. About the "critbit" lookup: To enable this, use "-hcritbit" argument. "Crit Bit" trees, are also known under various other names, the original version of the idea is probably the PATRICIA tree. The basic concept is a tree structure which has nodes only where necessary to tell the indices apart. Our version of it, has some additional bells and whistles. First lookups do not require any locks until we reach the objhead we were looking for, or until we need to insert one which wasn't there. Second, the branch nodes are part of the objhead, as all but the very first will need one, this saves malloc operations big time. Now the down-sides: There are still missing bits, amongst these the "cooling off" list, for objheads that have been dereferenced, but where the branch-node is not. Currently we just leak that memory. There is a race relating to node deref and unlocked lookup that is not closed, weird things may happen until I fix it. I'd be interested to hear how long it survives before it croaks, but apart from that, would not advocate that you use it, until I fix those remaining issues. git-svn-id: svn+ssh://projects.linpro.no/svn/varnish/trunk@3454 d4fa192b-c00b-0410-8231-f00ffab90ce4 --- diff --git a/varnish-cache/bin/varnishd/Makefile.am b/varnish-cache/bin/varnishd/Makefile.am index acbf581b..278a9cfc 100644 --- a/varnish-cache/bin/varnishd/Makefile.am +++ b/varnish-cache/bin/varnishd/Makefile.am @@ -40,6 +40,7 @@ varnishd_SOURCES = \ cache_vrt_re.c \ cache_ws.c \ hash_classic.c \ + hash_critbit.c \ hash_simple_list.c \ instance.c \ mgt_child.c \ diff --git a/varnish-cache/bin/varnishd/cache_hash.c b/varnish-cache/bin/varnishd/cache_hash.c index 8776ee56..e756f814 100644 --- a/varnish-cache/bin/varnishd/cache_hash.c +++ b/varnish-cache/bin/varnishd/cache_hash.c @@ -251,7 +251,6 @@ HSH_Lookup(struct sess *sp) struct http *h; struct objhead *oh; struct object *o, *busy_o, *grace_o; - unsigned char sha256[32]; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); @@ -259,12 +258,14 @@ HSH_Lookup(struct sess *sp) AN(hash); w = sp->wrk; h = sp->http; + + HSH_Prealloc(sp); if (params->hash_sha256) { - SHA256_Final(sha256, sp->wrk->sha256ctx); + SHA256_Final(sp->wrk->nobjhead->digest, sp->wrk->sha256ctx); + sp->wrk->nobjhead->digest_len = 32; /* WSP(sp, SLT_Debug, "SHA256: <%.32s>", sha256); */ } - - HSH_Prealloc(sp); + if (sp->objhead != NULL) { CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC); oh = sp->objhead; diff --git a/varnish-cache/bin/varnishd/hash_slinger.h b/varnish-cache/bin/varnishd/hash_slinger.h index 91ff69b6..90f55af5 100644 --- a/varnish-cache/bin/varnishd/hash_slinger.h +++ b/varnish-cache/bin/varnishd/hash_slinger.h @@ -73,14 +73,24 @@ struct objhead { VTAILQ_HEAD(,object) objects; char *hash; unsigned hashlen; + unsigned char digest[32]; + unsigned char digest_len; VTAILQ_HEAD(, sess) waitinglist; - /*------------------------------------------------------------ - * The fields below are for the sole private use of the hash - * implementation. + /*---------------------------------------------------- + * The fields below are for the sole private use of + * the hash implementation(s). */ - VTAILQ_ENTRY(objhead) hoh_list; - void *hoh_head; - unsigned hoh_digest; + union { + void *filler[3]; + struct { + VTAILQ_ENTRY(objhead) u_n_hoh_list; + void *u_n_hoh_head; + unsigned u_n_hoh_digest; + } n; + } u; +#define hoh_list u.n.u_n_hoh_list +#define hoh_head u.n.u_n_hoh_head +#define hoh_digest u.n.u_n_hoh_digest }; #endif /* VARNISH_CACHE_CHILD */ diff --git a/varnish-cache/bin/varnishd/mgt_param.c b/varnish-cache/bin/varnishd/mgt_param.c index 6a621d9f..923e7603 100644 --- a/varnish-cache/bin/varnishd/mgt_param.c +++ b/varnish-cache/bin/varnishd/mgt_param.c @@ -713,7 +713,7 @@ static const struct parspec input_parspec[] = { { "hash_sha256", tweak_bool, &master.hash_sha256, 0, 0, "Use SHA256 compression of hash-strings", 0, - "off", "bool" }, + "on", "bool" }, { "log_hashstring", tweak_bool, &master.log_hash, 0, 0, "Log the hash string to shared memory log.\n", 0, diff --git a/varnish-cache/bin/varnishd/varnishd.c b/varnish-cache/bin/varnishd/varnishd.c index b2ffd2b8..ce53ed9a 100644 --- a/varnish-cache/bin/varnishd/varnishd.c +++ b/varnish-cache/bin/varnishd/varnishd.c @@ -59,6 +59,7 @@ #include "vsb.h" #include "vpf.h" +#include "vsha256.h" #include "cli.h" #include "cli_priv.h" @@ -157,11 +158,13 @@ setup_storage(const char *spec) extern struct hash_slinger hsl_slinger; extern struct hash_slinger hcl_slinger; +extern struct hash_slinger hcb_slinger; static const struct choice hsh_choice[] = { { "classic", &hcl_slinger }, { "simple", &hsl_slinger }, { "simple_list", &hsl_slinger }, /* backwards compat */ + { "critbit", &hcb_slinger }, { NULL, NULL } }; @@ -450,6 +453,11 @@ main(int argc, char * const *argv) assert(TIM_parse("Sunday, 06-Nov-94 08:49:37 GMT") == 784111777); assert(TIM_parse("Sun Nov 6 08:49:37 1994") == 784111777); + /* + * Check that our SHA256 works + */ + SHA256_Test(); + memset(cli, 0, sizeof cli); cli[0].sb = vsb_newauto(); XXXAN(cli[0].sb); diff --git a/varnish-cache/include/stat_field.h b/varnish-cache/include/stat_field.h index 127e30b9..8516b24b 100644 --- a/varnish-cache/include/stat_field.h +++ b/varnish-cache/include/stat_field.h @@ -126,3 +126,7 @@ MAC_STAT(n_purge_retire, uint64_t, 'a', "N old purges deleted") MAC_STAT(n_purge_obj_test, uint64_t, 'a', "N objects tested") MAC_STAT(n_purge_re_test, uint64_t, 'a', "N regexps tested against") MAC_STAT(n_purge_dups, uint64_t, 'a', "N duplicate purges removed") + +MAC_STAT(hcb_nolock, uint64_t, 'a', "HCB Lookups without lock") +MAC_STAT(hcb_lock, uint64_t, 'a', "HCB Lookups with lock") +MAC_STAT(hcb_insert, uint64_t, 'a', "HCB Inserts")