Add per address, per session and total statistics.

We (will) collect data in unlocked per workerthread accumulators
and whenever the workerthread leaves the session, we charge the
bill to the srcaddr (issuing a StatAddr shmrecord), to the session
and to the global counters in the stats struct.

When sessions die we issue a StatSess shmrecord.

StatAddr and StatSess has the same format:
	address
	port (always zero for StatAddr)
	duration (seconds)
	#sessions
	#requests
	#pipe
	#pass
	#fetch
	#hdrbytes
	#bodybytes

git-svn-id: svn+ssh://projects.linpro.no/svn/varnish/trunk@564 d4fa192b-c00b-0410-8231-f00ffab90ce4

diff --git a/varnish-cache/bin/varnishd/cache.h b/varnish-cache/bin/varnishd/cache.h
index ce289610627fa5b3b91b36c51190b947ebd5d48f..48cf9b63a567df4e939943bf82910da0a74b5660 100644 (file)
--- a/varnish-cache/bin/varnishd/cache.h
+++ b/varnish-cache/bin/varnishd/cache.h
@@ -74,6 +74,19 @@ struct http {
 
 /*--------------------------------------------------------------------*/
 
+struct acct {
+       time_t                  first;
+       uint64_t                sess;
+       uint64_t                req;
+       uint64_t                pipe;
+       uint64_t                pass;
+       uint64_t                fetch;
+       uint64_t                hdrbytes;
+       uint64_t                bodybytes;
+};
+
+/*--------------------------------------------------------------------*/
+
 struct worker {
        unsigned                magic;
 #define WORKER_MAGIC           0x6391adcf
@@ -90,6 +103,8 @@ struct worker {
        struct iovec            iov[MAX_IOVS];
        unsigned                niov;
        size_t                  liov;
+
+       struct acct             acct;
 };
 
 struct workreq {
@@ -189,20 +204,24 @@ struct objhead {
 struct srcaddr {
        unsigned                magic;
 #define SRCADDR_MAGIC          0x375111db
+
+       unsigned                hash;
        TAILQ_ENTRY(srcaddr)    list;
-       unsigned                nsess;
+       struct srcaddrhead      *sah;
+
        char                    addr[TCP_ADDRBUFSIZE];
-       unsigned                sum;
-       time_t                  first;
+       unsigned                nref;
+
        time_t                  ttl;
-       uint64_t                bytes;
-       struct srcaddrhead      *sah;
+
+       struct acct             acct;
 };
 
 struct sess {
        unsigned                magic;
 #define SESS_MAGIC             0x2c2f9c5a
        int                     fd;
+       int                     id;
        unsigned                xid;
 
        struct worker           *wrk;
@@ -237,6 +256,7 @@ struct sess {
        time_t                  t0;
 
        struct workreq          workreq;
+       struct acct             acct;
 };
 
 struct backend {
@@ -344,7 +364,7 @@ void SES_Init(void);
 struct sess *SES_New(struct sockaddr *addr, unsigned len);
 void SES_Delete(struct sess *sp);
 void SES_RefSrcAddr(struct sess *sp);
-void SES_ChargeBytes(struct sess *sp, uint64_t bytes);
+void SES_Charge(struct sess *sp);
 
 /* cache_shmlog.c */
 

diff --git a/varnish-cache/bin/varnishd/cache_acceptor.c b/varnish-cache/bin/varnishd/cache_acceptor.c
index a6a890f3dde7d9b3e33fb85421c0d652dc5d9510..840f6abd47eb1cf6e03ce88cc3e13b9ee2d5e13a 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_acceptor.c
+++ b/varnish-cache/bin/varnishd/cache_acceptor.c
@@ -118,6 +118,7 @@ accept_f(int fd, short event, void *arg)
        assert(sp != NULL);     /* XXX handle */
 
        sp->fd = i;
+       sp->id = i;
 
 #ifdef SO_NOSIGPIPE /* XXX Linux */
        i = 1;

diff --git a/varnish-cache/bin/varnishd/cache_center.c b/varnish-cache/bin/varnishd/cache_center.c
index 90737bdea33c730e183da4a29f2d1a6b6b1c1792..f967afe6a80e166144b8c96588a8278c571aff82 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_center.c
+++ b/varnish-cache/bin/varnishd/cache_center.c
@@ -89,6 +89,7 @@ cnt_done(struct sess *sp)
        VCL_Rel(sp->vcl);
        sp->vcl = NULL;
 
+       SES_Charge(sp);
        vca_return_session(sp);
        return (1);
 }
@@ -316,6 +317,7 @@ cnt_lookup2(struct sess *sp)
        if (o == NULL) {
                VSL(SLT_Debug, sp->fd,
                    "on waiting list on obj %u", sp->obj->xid);
+               SES_Charge(sp);
                return (1);
        }
 
@@ -584,7 +586,7 @@ CNT_Session(struct sess *sp)
                switch (sp->step) {
 #define STEP(l,u) \
                case STP_##u: \
-                       VSL(SLT_Debug, sp->fd, "State " #u); \
+                       VSL(SLT_Debug, sp->id, "State " #u); \
                        done = cnt_##l(sp); \
                        break;
 #include "steps.h"

diff --git a/varnish-cache/bin/varnishd/cache_pool.c b/varnish-cache/bin/varnishd/cache_pool.c
index cd7c3e882857b548daae4bf4d7752a1f3482277d..1ed8f565b3a20df5e7069a3518760f12561d6f79 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_pool.c
+++ b/varnish-cache/bin/varnishd/cache_pool.c
@@ -105,8 +105,10 @@ wrk_do_one(struct worker *w)
        AZ(pthread_mutex_unlock(&wrk_mtx));
        CHECK_OBJ_NOTNULL(wrq->sess, SESS_MAGIC);
        wrq->sess->wrk = w;
-       if (wrq->sess->srcaddr == NULL)
+       if (wrq->sess->srcaddr == NULL) {
+               w->acct.sess++;
                SES_RefSrcAddr(wrq->sess);
+       }
        if (w->nobj != NULL)
                CHECK_OBJ(w->nobj, OBJECT_MAGIC);
        if (w->nobjhead != NULL)

diff --git a/varnish-cache/bin/varnishd/cache_response.c b/varnish-cache/bin/varnishd/cache_response.c
index f8e8ebdf5ae0028f63c9e052d4178b8552d569c5..107ae74f7c063106e1ca3e8b792c04c9a9364768 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_response.c
+++ b/varnish-cache/bin/varnishd/cache_response.c
@@ -138,7 +138,6 @@ RES_WriteObj(struct sess *sp)
 {
        struct storage *st;
        unsigned u = 0;
-       uint64_t bytes = 0;
        
        if (sp->obj->response == 200 && sp->http->conds && res_do_conds(sp))
                return;
@@ -162,9 +161,6 @@ RES_WriteObj(struct sess *sp)
        WRK_Reset(sp->wrk, &sp->fd);
        http_Write(sp->wrk, sp->http, 1);
        
-#if 0 /* XXX */
-       bytes += sbuf_len(sb);
-#endif
        /* XXX: conditional request handling */
        if (!strcmp(sp->http->hd[HTTP_HDR_REQ].b, "GET")) {
                TAILQ_FOREACH(st, &sp->obj->store, list) {
@@ -181,7 +177,6 @@ RES_WriteObj(struct sess *sp)
                }
                assert(u == sp->obj->len);
        }
-       SES_ChargeBytes(sp, bytes + u);
        if (WRK_Flush(sp->wrk))
                vca_close_session(sp, "remote closed");
 }

diff --git a/varnish-cache/bin/varnishd/cache_session.c b/varnish-cache/bin/varnishd/cache_session.c
index 1fffbbe1534b4e0a5ad3a07f23c7ba0b71d0efc1..a4df101fa07800533a2dab58b1e8e76546434c17 100644 (file)
--- a/varnish-cache/bin/varnishd/cache_session.c
+++ b/varnish-cache/bin/varnishd/cache_session.c
@@ -74,10 +74,10 @@ SES_RefSrcAddr(struct sess *sp)
        AZ(pthread_mutex_lock(&ses_mtx));
        c3 = NULL;
        TAILQ_FOREACH_SAFE(c, ch, list, c2) {
-               if (c->sum == u && !strcmp(c->addr, sp->addr)) {
-                       if (c->nsess == 0)
+               if (c->hash == u && !strcmp(c->addr, sp->addr)) {
+                       if (c->nref == 0)
                                VSL_stats->n_srcaddr_act++;
-                       c->nsess++;
+                       c->nref++;
                        c->ttl = now + CLIENT_TTL;
                        sp->srcaddr = c;
                        TAILQ_REMOVE(ch, c, list);
@@ -90,7 +90,7 @@ SES_RefSrcAddr(struct sess *sp)
                        AZ(pthread_mutex_unlock(&ses_mtx));
                        return;
                }
-               if (c->nsess > 0 || c->ttl > now)
+               if (c->nref > 0 || c->ttl > now)
                        continue;
                if (c3 == NULL) {
                        c3 = c;
@@ -111,10 +111,10 @@ SES_RefSrcAddr(struct sess *sp)
        if (c3 != NULL) {
                memset(c3, 0, sizeof *c3);
                strcpy(c3->addr, sp->addr);
-               c3->sum = u;
-               c3->first = now;
+               c3->hash = u;
+               c3->acct.first = now;
                c3->ttl = now + CLIENT_TTL;
-               c3->nsess = 1;
+               c3->nref = 1;
                c3->sah = ch;
                VSL_stats->n_srcaddr_act++;
                TAILQ_INSERT_TAIL(ch, c3, list);
@@ -123,24 +123,42 @@ SES_RefSrcAddr(struct sess *sp)
        AZ(pthread_mutex_unlock(&ses_mtx));
 }
 
+static void
+ses_sum_acct(struct acct *sum, struct acct *inc)
+{
+
+       sum->sess += inc->sess;
+       sum->req += inc->req;
+       sum->pipe += inc->pipe;
+       sum->pass += inc->pass;
+       sum->fetch += inc->fetch;
+       sum->hdrbytes += inc->hdrbytes;
+       sum->bodybytes += inc->bodybytes;
+}
+
 void
-SES_ChargeBytes(struct sess *sp, uint64_t bytes)
+SES_Charge(struct sess *sp)
 {
-       struct srcaddr *sa;
-       time_t now;
+       struct acct *a = &sp->wrk->acct;
+       struct acct *b = &sp->srcaddr->acct;
 
-       assert(sp->srcaddr != NULL);
-       sa = sp->srcaddr;
-       now = time(NULL);
+       ses_sum_acct(&sp->acct, a);
+       
        AZ(pthread_mutex_lock(&ses_mtx));
-       sa->bytes += bytes;
-       sa->ttl = now + CLIENT_TTL;
-       TAILQ_REMOVE(sa->sah, sa, list);
-       TAILQ_INSERT_TAIL(sa->sah, sa, list);
-       bytes = sa->bytes;
+       ses_sum_acct(b, a);
+       VSL(SLT_StatAddr, sp->id, "%s 0 %d %ju %ju %ju %ju %ju %ju %ju",
+           sp->srcaddr->addr, time(NULL) - b->first,
+           b->sess, b->req, b->pipe, b->pass,
+           b->fetch, b->hdrbytes, b->bodybytes);
+       VSL_stats->s_sess += a->sess;
+       VSL_stats->s_req += a->req;
+       VSL_stats->s_pipe += a->pipe;
+       VSL_stats->s_pass += a->pass;
+       VSL_stats->s_fetch += a->fetch;
+       VSL_stats->s_hdrbytes += a->hdrbytes;
+       VSL_stats->s_bodybytes += a->bodybytes;
        AZ(pthread_mutex_unlock(&ses_mtx));
-       VSL(SLT_SrcAddr, sp->fd, "%s %jd %d",
-           sa->addr, (intmax_t)(bytes), now - sa->first);
+       memset(a, 0, sizeof *a);
 }
 
 static void
@@ -153,9 +171,9 @@ ses_relsrcaddr(struct sess *sp)
        }
        assert(sp->srcaddr != NULL);
        AZ(pthread_mutex_lock(&ses_mtx));
-       assert(sp->srcaddr->nsess > 0);
-       sp->srcaddr->nsess--;
-       if (sp->srcaddr->nsess == 0)
+       assert(sp->srcaddr->nref > 0);
+       sp->srcaddr->nref--;
+       if (sp->srcaddr->nref == 0)
                VSL_stats->n_srcaddr_act--;
        sp->srcaddr = NULL;
        AZ(pthread_mutex_unlock(&ses_mtx));
@@ -183,17 +201,26 @@ SES_New(struct sockaddr *addr, unsigned len)
        assert(len  < sizeof(sm->sockaddr));
        memcpy(sm->sess.sockaddr, addr, len);
        sm->sess.sockaddrlen = len;
+
        http_Setup(&sm->http, (void *)(sm + 1), heritage.mem_workspace);
+
+       sm->sess.acct.first = time(NULL);
+
        return (&sm->sess);
 }
 
 void
 SES_Delete(struct sess *sp)
 {
+       struct acct *b = &sp->acct;
 
        CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
        VSL_stats->n_sess--;
        ses_relsrcaddr(sp);
+       VSL(SLT_StatSess, sp->id, "%s %s %d %ju %ju %ju %ju %ju %ju %ju",
+           sp->addr, sp->port, time(NULL) - b->first,
+           b->sess, b->req, b->pipe, b->pass,
+           b->fetch, b->hdrbytes, b->bodybytes);
        CHECK_OBJ_NOTNULL(sp->mem, SESSMEM_MAGIC);
        free(sp->mem);
 }

diff --git a/varnish-cache/include/shmlog_tags.h b/varnish-cache/include/shmlog_tags.h
index 176eb364458aa694a607a09164462c3c929a86da..a77ac2de34e448323f5da47a93d2a9eef55f1cd0 100644 (file)
--- a/varnish-cache/include/shmlog_tags.h
+++ b/varnish-cache/include/shmlog_tags.h
@@ -11,7 +11,8 @@
 SLTM(Debug)
 SLTM(Error)
 SLTM(CLI)
-SLTM(SrcAddr)
+SLTM(StatAddr)
+SLTM(StatSess)
 SLTM(SessionOpen)
 SLTM(SessionReuse)
 SLTM(SessionClose)

diff --git a/varnish-cache/include/stat_field.h b/varnish-cache/include/stat_field.h
index 78f85bc2f2c8ba85026c74da51b65f4d3317b3ff..03b0bd2e0d8d87f92ee14b52010a012e74272422 100644 (file)
--- a/varnish-cache/include/stat_field.h
+++ b/varnish-cache/include/stat_field.h
@@ -29,3 +29,11 @@ MAC_STAT(n_expired,          uint64_t, "u", "N expired objects")
 MAC_STAT(n_deathrow,           uint64_t, "u", "N objects on deathrow")
 
 MAC_STAT(losthdr,              uint64_t, "u", "HTTP header overflows")
+
+MAC_STAT(s_sess,               uint64_t, "u", "Total Sessions")
+MAC_STAT(s_req,                        uint64_t, "u", "Total Requests")
+MAC_STAT(s_pipe,               uint64_t, "u", "Total pipe")
+MAC_STAT(s_pass,               uint64_t, "u", "Total pass")
+MAC_STAT(s_fetch,              uint64_t, "u", "Total fetch")
+MAC_STAT(s_hdrbytes,           uint64_t, "u", "Total header bytes")
+MAC_STAT(s_bodybytes,          uint64_t, "u", "Total body bytes")