Merge r3329, r3330: Add a boolean parameter "purge_dups"

Add a boolean parameter "purge_dups", to make it possible to mark earlier
bans of the same regexp as "gone" to save duplicate regexp checking.

Prodded to by: jodok

Add a test-case for dup purge elimination

git-svn-id: svn+ssh://projects.linpro.no/svn/varnish/branches/2.0@3370 d4fa192b-c00b-0410-8231-f00ffab90ce4

diff --git a/varnish-cache/bin/varnishd/cache_ban.c b/varnish-cache/bin/varnishd/cache_ban.c
index 050d09834db192465de591333480c39cf8539d2d..3475de5919a6f89a81c999ed80d7dd17a994e60b 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_ban.c
+++ b/varnish-cache/bin/varnishd/cache_ban.c
@@ -49,6 +49,8 @@ struct ban {
 #define BAN_MAGIC              0x700b08ea
        VTAILQ_ENTRY(ban)       list;
        unsigned                refcount;
+       int                     flags;
+#define BAN_F_GONE             (1 << 0)
        regex_t                 regexp;
        char                    *ban;
        int                     hash;
@@ -68,8 +70,9 @@ static struct ban * volatile ban_start;
 int
 BAN_Add(struct cli *cli, const char *regexp, int hash)
 {
-       struct ban *b;
+       struct ban *b, *bi, *be;
        char buf[512];
+       unsigned pcount;
        int i;
 
        ALLOC_OBJ(b, BAN_MAGIC);
@@ -97,6 +100,35 @@ BAN_Add(struct cli *cli, const char *regexp, int hash)
        ban_start = b;
        VSL_stats->n_purge++;
        VSL_stats->n_purge_add++;
+
+       if (params->purge_dups) {
+               be = VTAILQ_LAST(&ban_head, banhead);
+               be->refcount++;
+       } else
+               be = NULL;
+       UNLOCK(&ban_mtx);
+
+       if (be == NULL)
+               return (0);
+
+       /* Hunt down duplicates, and mark them as gone */
+       bi = b;
+       pcount = 0;
+       while(bi != be) {
+               bi = VTAILQ_NEXT(bi, list);
+               if (bi->flags & BAN_F_GONE)
+                       continue;
+               if (b->hash != bi->hash)
+                       continue;
+               if (strcmp(b->ban, bi->ban))
+                       continue;
+               bi->flags |= BAN_F_GONE;
+               pcount++;
+       }
+       LOCK(&ban_mtx);
+       be->refcount--;
+       /* XXX: We should check if the tail can be removed */
+       VSL_stats->n_purge_dups += pcount;
        UNLOCK(&ban_mtx);
 
        return (0);
@@ -168,7 +200,8 @@ BAN_CheckObject(struct object *o, const char *url, const char *hash)
        tests = 0;
        for (b = b0; b != o->ban; b = VTAILQ_NEXT(b, list)) {
                tests++;
-               if (!regexec(&b->regexp, b->hash ? hash : url, 0, NULL, 0))
+               if (!(b->flags & BAN_F_GONE) &&
+                   !regexec(&b->regexp, b->hash ? hash : url, 0, NULL, 0))
                        break;
        }
 
@@ -227,8 +260,8 @@ ccf_purge_list(struct cli *cli, const char * const *av, void *priv)
        for (b0 = ban_start; b0 != NULL; b0 = VTAILQ_NEXT(b0, list)) {
                if (b0->refcount == 0 && VTAILQ_NEXT(b0, list) == NULL)
                        break;
-               cli_out(cli, "%5u %s \"%s\"\n",
-                   b0->refcount,
+               cli_out(cli, "%5u %d %s \"%s\"\n",
+                   b0->refcount, b0->flags,
                    b0->hash ? "hash" : "url ",
                    b0->ban);
        }

diff --git a/varnish-cache/bin/varnishd/heritage.h b/varnish-cache/bin/varnishd/heritage.h
index 9cf61337293b895d1859d439a092396c92ba8558..0e0c5ceb1480e71875fcaf2cbc532334f847c8ce 100644 (file)
--- a/varnish-cache/bin/varnishd/heritage.h
+++ b/varnish-cache/bin/varnishd/heritage.h
@@ -183,6 +183,9 @@ struct params {
        /* Amount of time to sleep when running out of file
           descriptors.  In msecs */
        unsigned                accept_fd_holdoff;
+
+       /* Get rid of duplicate purges */
+       unsigned                purge_dups;
 };
 
 extern volatile struct params *params;

diff --git a/varnish-cache/bin/varnishd/mgt_param.c b/varnish-cache/bin/varnishd/mgt_param.c
index fab995ddb3962419745d69ecc24e49c4d2d730ae..d1d1a126e23235a69402e57d8a7a0ca196207ef1 100644 (file)
--- a/varnish-cache/bin/varnishd/mgt_param.c
+++ b/varnish-cache/bin/varnishd/mgt_param.c
@@ -817,6 +817,10 @@ static const struct parspec parspec[] = {
                "The TTL assigned to the synthesized error pages\n",
                0,
                "0", "seconds" },
+       { "purge_dups", tweak_bool, &master.purge_dups, 0, 0,
+               "Detect and eliminate duplicate purges.\n",
+               0,
+               "off", "bool" },
        { NULL, NULL, NULL }
 };
 

diff --git a/varnish-cache/bin/varnishtest/c00019.vtc b/varnish-cache/bin/varnishtest/c00019.vtc
new file mode 100644 (file)
index 0000000..87c2bd6
--- /dev/null
+++ b/varnish-cache/bin/varnishtest/c00019.vtc
@@ -0,0 +1,83 @@
+# $Id$
+
+test "Check purge counters and duplicate purge elimination"
+
+server s1 {
+       rxreq
+       txresp -hdr "foo: 1" -body "foo1"
+       rxreq
+       txresp -hdr "foo: 2" -body "foo2"
+       rxreq
+       txresp -hdr "foo: 3" -body "foo3"
+} -start
+
+varnish v1 -vcl+backend {} -start
+
+varnish v1 -cliok "purge.url FOO"
+
+# There is one "magic" purge from boot
+varnish v1 -expect n_purge_add == 2
+varnish v1 -cliok "purge.list"
+
+# Our fetch is not affected by the purge
+# as the FOO-purge was preexisting
+client c1 {
+       txreq -url /FOO
+       rxresp
+       expect resp.http.foo == 1
+} -run
+
+varnish v1 -cliok "purge.list"
+varnish v1 -expect n_purge_obj_test == 0
+varnish v1 -expect n_purge_re_test == 0
+
+# Add another purge
+varnish v1 -cliok "purge.url FOO"
+varnish v1 -expect n_purge_add == 3
+varnish v1 -cliok "purge.list"
+
+# The cached object will be purged, and a new
+# fetched from the backend
+client c1 {
+       txreq -url /FOO
+       rxresp
+       expect resp.http.foo == 2
+} -run
+
+varnish v1 -expect n_purge_obj_test == 1
+varnish v1 -expect n_purge_re_test == 1
+varnish v1 -cliok "purge.list"
+
+# Fetch the cached copy, just for grins
+client c1 {
+       txreq -url /FOO
+       rxresp
+       expect resp.http.foo == 2
+} -run
+
+
+# Now add another purge
+varnish v1 -cliok "purge.url FOO"
+varnish v1 -expect n_purge_add == 4
+
+# Enable dup removal of purges
+varnish v1 -cliok "param.set purge_dups on"
+
+# This should incapacitate the to previous FOO purges.
+varnish v1 -cliok "purge.url FOO"
+varnish v1 -expect n_purge_add == 5
+varnish v1 -expect n_purge_dups == 3
+varnish v1 -cliok "purge.list"
+
+# And we should get a fresh object from backend
+client c1 {
+       txreq -url /FOO
+       rxresp
+       expect resp.http.foo == 3
+} -run
+
+# With only two objects having ever been compared
+varnish v1 -expect n_purge_obj_test == 2
+varnish v1 -expect n_purge_re_test == 2
+varnish v1 -cliok "purge.list"
+

diff --git a/varnish-cache/include/stat_field.h b/varnish-cache/include/stat_field.h
index 0ae472236682e649203acca1095641d99e250d39..ea7eb91f0cdb61617ecb663c5cdeb67845440bb9 100644 (file)
--- a/varnish-cache/include/stat_field.h
+++ b/varnish-cache/include/stat_field.h
@@ -124,3 +124,4 @@ MAC_STAT(n_purge_add,               uint64_t, 'a', "N new purges added")
 MAC_STAT(n_purge_retire,       uint64_t, 'a', "N old purges deleted")
 MAC_STAT(n_purge_obj_test,     uint64_t, 'a', "N objects tested")
 MAC_STAT(n_purge_re_test,      uint64_t, 'a', "N regexps tested against")
+MAC_STAT(n_purge_dups,         uint64_t, 'a', "N duplicate purges removed")