Add preliminary version of lock-less tree based lookup (see below)

Enable SHA256 digests by default, and put it in the objhead.  This
increases the size of the objhead by 32 bytes, but may drop
a bit again later, when other now unnecessary fields go away.

Test SHA256 for correct operation on startup.

About the "critbit" lookup:

To enable this, use "-hcritbit" argument.

"Crit Bit" trees, are also known under various other names, the original
version of the idea is probably the PATRICIA tree.

The basic concept is a tree structure which has nodes only where necessary
to tell the indices apart.

Our version of it, has some additional bells and whistles.

First lookups do not require any locks until we reach the objhead
we were looking for, or until we need to insert one which wasn't
there.

Second, the branch nodes are part of the objhead, as all but the
very first will need one, this saves malloc operations big time.

Now the down-sides:

There are still missing bits, amongst these the "cooling off" list,
for objheads that have been dereferenced, but where the branch-node
is not.  Currently we just leak that memory.

There is a race relating to node deref and unlocked lookup that is
not closed, weird things may happen until I fix it.

I'd be interested to hear how long it survives before it croaks,
but apart from that, would not advocate that you use it, until
I fix those remaining issues.

git-svn-id: svn+ssh://projects.linpro.no/svn/varnish/trunk@3454 d4fa192b-c00b-0410-8231-f00ffab90ce4

diff --git a/varnish-cache/bin/varnishd/Makefile.am b/varnish-cache/bin/varnishd/Makefile.am
index acbf581bd26f4ddbf2dc2f051a7a7cf01159e918..278a9cfc267fea6efbe4381a409827051613a18d 100644 (file)
--- a/varnish-cache/bin/varnishd/Makefile.am
+++ b/varnish-cache/bin/varnishd/Makefile.am
@@ -40,6 +40,7 @@ varnishd_SOURCES = \
        cache_vrt_re.c \
        cache_ws.c \
        hash_classic.c \
+       hash_critbit.c \
        hash_simple_list.c \
        instance.c \
        mgt_child.c \

diff --git a/varnish-cache/bin/varnishd/cache_hash.c b/varnish-cache/bin/varnishd/cache_hash.c
index 8776ee5661c33c9f5f6b071762e9653496d413ed..e756f814e345035776e5fcec7b488976c7ac2873 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_hash.c
+++ b/varnish-cache/bin/varnishd/cache_hash.c
@@ -251,7 +251,6 @@ HSH_Lookup(struct sess *sp)
        struct http *h;
        struct objhead *oh;
        struct object *o, *busy_o, *grace_o;
-       unsigned char sha256[32];
 
        CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
        CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
@@ -259,12 +258,14 @@ HSH_Lookup(struct sess *sp)
        AN(hash);
        w = sp->wrk;
        h = sp->http;
+
+       HSH_Prealloc(sp);
        if (params->hash_sha256) {
-               SHA256_Final(sha256, sp->wrk->sha256ctx);
+               SHA256_Final(sp->wrk->nobjhead->digest, sp->wrk->sha256ctx);
+               sp->wrk->nobjhead->digest_len = 32;
                /* WSP(sp, SLT_Debug, "SHA256: <%.32s>", sha256); */
        }
-
-       HSH_Prealloc(sp);
+       
        if (sp->objhead != NULL) {
                CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC);
                oh = sp->objhead;

diff --git a/varnish-cache/bin/varnishd/hash_slinger.h b/varnish-cache/bin/varnishd/hash_slinger.h
index 91ff69b68259381678ec30756cfdd368952cee48..90f55af558a85908815ac613ab6004c983284df9 100644 (file)
--- a/varnish-cache/bin/varnishd/hash_slinger.h
+++ b/varnish-cache/bin/varnishd/hash_slinger.h
@@ -73,14 +73,24 @@ struct objhead {
        VTAILQ_HEAD(,object)    objects;
        char                    *hash;
        unsigned                hashlen;
+       unsigned char           digest[32];
+       unsigned char           digest_len;
        VTAILQ_HEAD(, sess)     waitinglist;
 
-       /*------------------------------------------------------------
-        * The fields below are for the sole private use of the hash 
-        * implementation.
+       /*----------------------------------------------------
+        * The fields below are for the sole private use of
+        * the hash implementation(s).
         */
-       VTAILQ_ENTRY(objhead)   hoh_list;
-       void                    *hoh_head;
-       unsigned                hoh_digest;
+       union {
+               void            *filler[3];
+               struct {
+                       VTAILQ_ENTRY(objhead)   u_n_hoh_list;
+                       void                    *u_n_hoh_head;
+                       unsigned                u_n_hoh_digest;
+               } n;
+       } u;
+#define hoh_list u.n.u_n_hoh_list
+#define hoh_head u.n.u_n_hoh_head
+#define hoh_digest u.n.u_n_hoh_digest
 };
 #endif /* VARNISH_CACHE_CHILD */

diff --git a/varnish-cache/bin/varnishd/mgt_param.c b/varnish-cache/bin/varnishd/mgt_param.c
index 6a621d9fbd4d7d2ea35dbfbd53307549d95c0439..923e7603759bdd406cd78ab6ee991515c415a91d 100644 (file)
--- a/varnish-cache/bin/varnishd/mgt_param.c
+++ b/varnish-cache/bin/varnishd/mgt_param.c
@@ -713,7 +713,7 @@ static const struct parspec input_parspec[] = {
        { "hash_sha256", tweak_bool, &master.hash_sha256, 0, 0,
                "Use SHA256 compression of hash-strings",
                0,
-               "off", "bool" },
+               "on", "bool" },
        { "log_hashstring", tweak_bool, &master.log_hash, 0, 0,
                "Log the hash string to shared memory log.\n",
                0,

diff --git a/varnish-cache/bin/varnishd/varnishd.c b/varnish-cache/bin/varnishd/varnishd.c
index b2ffd2b8b616f0b20a982e2070addb4d1265d5e9..ce53ed9aeb89a1bd996a743b8b25c054657341e1 100644 (file)
--- a/varnish-cache/bin/varnishd/varnishd.c
+++ b/varnish-cache/bin/varnishd/varnishd.c
@@ -59,6 +59,7 @@
 
 #include "vsb.h"
 #include "vpf.h"
+#include "vsha256.h"
 
 #include "cli.h"
 #include "cli_priv.h"
@@ -157,11 +158,13 @@ setup_storage(const char *spec)
 
 extern struct hash_slinger hsl_slinger;
 extern struct hash_slinger hcl_slinger;
+extern struct hash_slinger hcb_slinger;
 
 static const struct choice hsh_choice[] = {
        { "classic",            &hcl_slinger },
        { "simple",             &hsl_slinger },
        { "simple_list",        &hsl_slinger }, /* backwards compat */
+       { "critbit",            &hcb_slinger },
        { NULL,                 NULL }
 };
 
@@ -450,6 +453,11 @@ main(int argc, char * const *argv)
        assert(TIM_parse("Sunday, 06-Nov-94 08:49:37 GMT") == 784111777);
        assert(TIM_parse("Sun Nov  6 08:49:37 1994") == 784111777);
 
+       /*
+        * Check that our SHA256 works
+        */
+       SHA256_Test();
+
        memset(cli, 0, sizeof cli);
        cli[0].sb = vsb_newauto();
        XXXAN(cli[0].sb);

diff --git a/varnish-cache/include/stat_field.h b/varnish-cache/include/stat_field.h
index 127e30b907b3cd26283a866507d99a9febaf211a..8516b24babaa85028e2a84ec7d4ffb7ac9064345 100644 (file)
--- a/varnish-cache/include/stat_field.h
+++ b/varnish-cache/include/stat_field.h
@@ -126,3 +126,7 @@ MAC_STAT(n_purge_retire,    uint64_t, 'a', "N old purges deleted")
 MAC_STAT(n_purge_obj_test,     uint64_t, 'a', "N objects tested")
 MAC_STAT(n_purge_re_test,      uint64_t, 'a', "N regexps tested against")
 MAC_STAT(n_purge_dups,         uint64_t, 'a', "N duplicate purges removed")
+
+MAC_STAT(hcb_nolock,           uint64_t, 'a', "HCB Lookups without lock")
+MAC_STAT(hcb_lock,             uint64_t, 'a', "HCB Lookups with lock")
+MAC_STAT(hcb_insert,           uint64_t, 'a', "HCB Inserts")