From a914e3c25733fe5feb4599f3ad80c349b777a710 Mon Sep 17 00:00:00 2001 From: tfheen Date: Mon, 10 Nov 2008 10:09:31 +0000 Subject: [PATCH] Merge r3329, r3330: Add a boolean parameter "purge_dups" Add a boolean parameter "purge_dups", to make it possible to mark earlier bans of the same regexp as "gone" to save duplicate regexp checking. Prodded to by: jodok Add a test-case for dup purge elimination git-svn-id: svn+ssh://projects.linpro.no/svn/varnish/branches/2.0@3370 d4fa192b-c00b-0410-8231-f00ffab90ce4 --- varnish-cache/bin/varnishd/cache_ban.c | 41 ++++++++++-- varnish-cache/bin/varnishd/heritage.h | 3 + varnish-cache/bin/varnishd/mgt_param.c | 4 ++ varnish-cache/bin/varnishtest/c00019.vtc | 83 ++++++++++++++++++++++++ varnish-cache/include/stat_field.h | 1 + 5 files changed, 128 insertions(+), 4 deletions(-) create mode 100644 varnish-cache/bin/varnishtest/c00019.vtc diff --git a/varnish-cache/bin/varnishd/cache_ban.c b/varnish-cache/bin/varnishd/cache_ban.c index 050d0983..3475de59 100644 --- a/varnish-cache/bin/varnishd/cache_ban.c +++ b/varnish-cache/bin/varnishd/cache_ban.c @@ -49,6 +49,8 @@ struct ban { #define BAN_MAGIC 0x700b08ea VTAILQ_ENTRY(ban) list; unsigned refcount; + int flags; +#define BAN_F_GONE (1 << 0) regex_t regexp; char *ban; int hash; @@ -68,8 +70,9 @@ static struct ban * volatile ban_start; int BAN_Add(struct cli *cli, const char *regexp, int hash) { - struct ban *b; + struct ban *b, *bi, *be; char buf[512]; + unsigned pcount; int i; ALLOC_OBJ(b, BAN_MAGIC); @@ -97,6 +100,35 @@ BAN_Add(struct cli *cli, const char *regexp, int hash) ban_start = b; VSL_stats->n_purge++; VSL_stats->n_purge_add++; + + if (params->purge_dups) { + be = VTAILQ_LAST(&ban_head, banhead); + be->refcount++; + } else + be = NULL; + UNLOCK(&ban_mtx); + + if (be == NULL) + return (0); + + /* Hunt down duplicates, and mark them as gone */ + bi = b; + pcount = 0; + while(bi != be) { + bi = VTAILQ_NEXT(bi, list); + if (bi->flags & BAN_F_GONE) + continue; + if (b->hash != bi->hash) + continue; + if (strcmp(b->ban, bi->ban)) + continue; + bi->flags |= BAN_F_GONE; + pcount++; + } + LOCK(&ban_mtx); + be->refcount--; + /* XXX: We should check if the tail can be removed */ + VSL_stats->n_purge_dups += pcount; UNLOCK(&ban_mtx); return (0); @@ -168,7 +200,8 @@ BAN_CheckObject(struct object *o, const char *url, const char *hash) tests = 0; for (b = b0; b != o->ban; b = VTAILQ_NEXT(b, list)) { tests++; - if (!regexec(&b->regexp, b->hash ? hash : url, 0, NULL, 0)) + if (!(b->flags & BAN_F_GONE) && + !regexec(&b->regexp, b->hash ? hash : url, 0, NULL, 0)) break; } @@ -227,8 +260,8 @@ ccf_purge_list(struct cli *cli, const char * const *av, void *priv) for (b0 = ban_start; b0 != NULL; b0 = VTAILQ_NEXT(b0, list)) { if (b0->refcount == 0 && VTAILQ_NEXT(b0, list) == NULL) break; - cli_out(cli, "%5u %s \"%s\"\n", - b0->refcount, + cli_out(cli, "%5u %d %s \"%s\"\n", + b0->refcount, b0->flags, b0->hash ? "hash" : "url ", b0->ban); } diff --git a/varnish-cache/bin/varnishd/heritage.h b/varnish-cache/bin/varnishd/heritage.h index 9cf61337..0e0c5ceb 100644 --- a/varnish-cache/bin/varnishd/heritage.h +++ b/varnish-cache/bin/varnishd/heritage.h @@ -183,6 +183,9 @@ struct params { /* Amount of time to sleep when running out of file descriptors. In msecs */ unsigned accept_fd_holdoff; + + /* Get rid of duplicate purges */ + unsigned purge_dups; }; extern volatile struct params *params; diff --git a/varnish-cache/bin/varnishd/mgt_param.c b/varnish-cache/bin/varnishd/mgt_param.c index fab995dd..d1d1a126 100644 --- a/varnish-cache/bin/varnishd/mgt_param.c +++ b/varnish-cache/bin/varnishd/mgt_param.c @@ -817,6 +817,10 @@ static const struct parspec parspec[] = { "The TTL assigned to the synthesized error pages\n", 0, "0", "seconds" }, + { "purge_dups", tweak_bool, &master.purge_dups, 0, 0, + "Detect and eliminate duplicate purges.\n", + 0, + "off", "bool" }, { NULL, NULL, NULL } }; diff --git a/varnish-cache/bin/varnishtest/c00019.vtc b/varnish-cache/bin/varnishtest/c00019.vtc new file mode 100644 index 00000000..87c2bd67 --- /dev/null +++ b/varnish-cache/bin/varnishtest/c00019.vtc @@ -0,0 +1,83 @@ +# $Id$ + +test "Check purge counters and duplicate purge elimination" + +server s1 { + rxreq + txresp -hdr "foo: 1" -body "foo1" + rxreq + txresp -hdr "foo: 2" -body "foo2" + rxreq + txresp -hdr "foo: 3" -body "foo3" +} -start + +varnish v1 -vcl+backend {} -start + +varnish v1 -cliok "purge.url FOO" + +# There is one "magic" purge from boot +varnish v1 -expect n_purge_add == 2 +varnish v1 -cliok "purge.list" + +# Our fetch is not affected by the purge +# as the FOO-purge was preexisting +client c1 { + txreq -url /FOO + rxresp + expect resp.http.foo == 1 +} -run + +varnish v1 -cliok "purge.list" +varnish v1 -expect n_purge_obj_test == 0 +varnish v1 -expect n_purge_re_test == 0 + +# Add another purge +varnish v1 -cliok "purge.url FOO" +varnish v1 -expect n_purge_add == 3 +varnish v1 -cliok "purge.list" + +# The cached object will be purged, and a new +# fetched from the backend +client c1 { + txreq -url /FOO + rxresp + expect resp.http.foo == 2 +} -run + +varnish v1 -expect n_purge_obj_test == 1 +varnish v1 -expect n_purge_re_test == 1 +varnish v1 -cliok "purge.list" + +# Fetch the cached copy, just for grins +client c1 { + txreq -url /FOO + rxresp + expect resp.http.foo == 2 +} -run + + +# Now add another purge +varnish v1 -cliok "purge.url FOO" +varnish v1 -expect n_purge_add == 4 + +# Enable dup removal of purges +varnish v1 -cliok "param.set purge_dups on" + +# This should incapacitate the to previous FOO purges. +varnish v1 -cliok "purge.url FOO" +varnish v1 -expect n_purge_add == 5 +varnish v1 -expect n_purge_dups == 3 +varnish v1 -cliok "purge.list" + +# And we should get a fresh object from backend +client c1 { + txreq -url /FOO + rxresp + expect resp.http.foo == 3 +} -run + +# With only two objects having ever been compared +varnish v1 -expect n_purge_obj_test == 2 +varnish v1 -expect n_purge_re_test == 2 +varnish v1 -cliok "purge.list" + diff --git a/varnish-cache/include/stat_field.h b/varnish-cache/include/stat_field.h index 0ae47223..ea7eb91f 100644 --- a/varnish-cache/include/stat_field.h +++ b/varnish-cache/include/stat_field.h @@ -124,3 +124,4 @@ MAC_STAT(n_purge_add, uint64_t, 'a', "N new purges added") MAC_STAT(n_purge_retire, uint64_t, 'a', "N old purges deleted") MAC_STAT(n_purge_obj_test, uint64_t, 'a', "N objects tested") MAC_STAT(n_purge_re_test, uint64_t, 'a', "N regexps tested against") +MAC_STAT(n_purge_dups, uint64_t, 'a', "N duplicate purges removed") -- 2.39.5