From: cecilihf Date: Tue, 26 Jun 2007 06:56:03 +0000 (+0000) Subject: Traffic replay tool. Parses varnish logs and attempts to recreate the traffic X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=825e4518317289235a20207b5cdbed7273ba632a;p=varnish Traffic replay tool. Parses varnish logs and attempts to recreate the traffic git-svn-id: svn+ssh://projects.linpro.no/svn/varnish/trunk@1574 d4fa192b-c00b-0410-8231-f00ffab90ce4 --- diff --git a/varnish-cache/bin/Makefile.am b/varnish-cache/bin/Makefile.am index 2ada23eb..5ffa64f0 100644 --- a/varnish-cache/bin/Makefile.am +++ b/varnish-cache/bin/Makefile.am @@ -1,3 +1,3 @@ # $Id$ -SUBDIRS = varnishadm varnishd varnishhist varnishlog varnishncsa varnishstat varnishtop +SUBDIRS = varnishadm varnishd varnishhist varnishlog varnishncsa varnishstat varnishtop varnishreplay diff --git a/varnish-cache/bin/varnishreplay/Makefile.am b/varnish-cache/bin/varnishreplay/Makefile.am new file mode 100644 index 00000000..24e2c8af --- /dev/null +++ b/varnish-cache/bin/varnishreplay/Makefile.am @@ -0,0 +1,17 @@ +# $Id: Makefile.am 1507 2007-06-10 11:46:05Z des $ + +INCLUDES = -I$(top_srcdir)/include + +bin_PROGRAMS = varnishreplay + +dist_man_MANS = varnishreplay.1 + +varnishreplay_SOURCES = \ + varnishreplay.c + +varnishreplay_CFLAGS = -include config.h + +varnishreplay_LDADD = \ + $(top_builddir)/lib/libvarnish/libvarnish.la \ + $(top_builddir)/lib/libcompat/libcompat.a \ + $(top_builddir)/lib/libvarnishapi/libvarnishapi.la diff --git a/varnish-cache/bin/varnishreplay/varnishreplay.1 b/varnish-cache/bin/varnishreplay/varnishreplay.1 new file mode 100644 index 00000000..097de279 --- /dev/null +++ b/varnish-cache/bin/varnishreplay/varnishreplay.1 @@ -0,0 +1,68 @@ +.\"- +.\" Copyright (c) 2006 Verdens Gang AS +.\" Copyright (c) 2006-2007 Linpro AS +.\" All rights reserved. +.\" +.\" Author: Cecilie Fritzvold +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" $Id$ +.\" +.Dd June 25, 2007 +.Dt VARNISHREPLAY 1 +.Os +.Sh NAME +.Nm varnishreplay +.Sh SYNOPSIS +.Nm +.Fl a Ar address:port +.Fl r Ar file +.Op Fl D +.Sh DESCRIPTION +The +.Nm +utility parses varnish logs and attempts to reproduce the traffic. +.Pp +The following options are available: +.Bl -tag -width Fl +.It Fl a Ar address:port +Send the traffic over tcp to this address and port. +This option is mandatory. +.It Fl r Ar file +Parse logs from this file. This option is mandatory. +.It Fl D +Turn on debugging mode. +.Sh SEE ALSO +.Xr varnishd 1 , +.Xr varnishhist 1 , +.Xr varnishncsa 1 , +.Xr varnishstat 1 , +.Xr varnishtop 1 +.Sh HISTORY +The +.Nm +utility was developed by +.An Cecilie Fritzvold Aq cecilihf@linpro.no . + +This manual page was written by +.An Cecilie Fritzvold Aq cecilihf@linpro.no . diff --git a/varnish-cache/bin/varnishreplay/varnishreplay.c b/varnish-cache/bin/varnishreplay/varnishreplay.c new file mode 100644 index 00000000..a22be7c1 --- /dev/null +++ b/varnish-cache/bin/varnishreplay/varnishreplay.c @@ -0,0 +1,555 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2007 Linpro AS + * All rights reserved. + * + * Author: Cecilie Fritzvold + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id$ + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "libvarnish.h" +#include "varnishapi.h" +#include "vss.h" + +static struct request { + char *df_H; /* %H, Protocol version */ + char *df_Host; /* %{Host}i */ + char *df_Uq; /* %U%q, URL path and query string */ + char *df_m; /* %m, Request method*/ + char *df_c; /* Connection info (keep-alive, close) */ + int bogus; /* bogus request */ +} **req; + +static size_t nreq; + +static struct vss_addr *adr_info; +static int sock; +static int reopen; +static int debug; + +static int +isprefix(const char *str, const char *prefix, const char *end, const char **next) +{ + + while (str < end && *str && *prefix && + tolower((int)*str) == tolower((int)*prefix)) + ++str, ++prefix; + if (*str && *str != ' ') + return (0); + if (next) { + while (str < end && *str && *str == ' ') + ++str; + *next = str; + } + return (1); +} + +static int +isequal(const char *str, const char *reference, const char *end) +{ + + while (str < end && *str && *reference && + tolower((int)*str) == tolower((int)*reference)) + ++str, ++reference; + if (str != end || *reference) + return (0); + return (1); +} + +/* + * Returns a copy of the entire string with leading and trailing spaces + * trimmed. + */ +static char * +trimline(const char *str, const char *end) +{ + size_t len; + char *p; + + /* skip leading space */ + while (str < end && *str && *str == ' ') + ++str; + + /* seek to end of string */ + for (len = 0; &str[len] < end && str[len]; ++len) + /* nothing */ ; + + /* trim trailing space */ + while (len && str[len - 1] == ' ') + --len; + + /* copy and return */ + p = malloc(len + 1); + assert(p != NULL); + memcpy(p, str, len); + p[len] = '\0'; + return (p); +} + +/* Initiate a connection to
by resolving the + * hostname and returning a struct with necessary + * connection info. + */ +static struct vss_addr* +init_connection(const char* address) +{ + struct vss_addr **ta; + struct vss_addr *tap; + char *addr, *port; + int i, n; + + XXXAZ(VSS_parse(address, &addr, &port)); + XXXAN(n = VSS_resolve(addr, port, &ta)); + free(addr); + free(port); + if (n == 0) { + fprintf(stderr, "Could not open TELNET port\n"); + exit(2); + } + for (i = 1; i < n; ++i) { + free(ta[i]); + ta[i] = NULL; + } + tap = ta[0]; + free(ta); + + return tap; +} + +/* Read a line from the socket and return the number of bytes read. + * After returning, line will point to the read bytes in memory. + * A line is terminated by \r\n + */ +static int +read_line(char **line) +{ + char *buf; + unsigned nbuf, lbuf; + int i; + + lbuf = 4096; + buf = malloc(lbuf); + XXXAN(buf); + nbuf = 0; + while (1) { + if ((nbuf + 2) >= lbuf) { + lbuf += lbuf; + buf = realloc(buf, lbuf); + XXXAN(buf); + } + //fprintf(stderr, "start reading\n"); + i = read(sock, buf + nbuf, 1); + if (i <= 0) { + perror("error in reading\n"); + free(buf); + exit(1); + } + nbuf += i; + if (buf[nbuf-2] == '\r' && buf[nbuf-1] == '\n') + break; + + } + buf[nbuf] = '\0'; + *line = buf; + return nbuf+1; +} + +/* Read a block of data from the socket, and do nothing with it. + * length says how many bytes to read, and the function returns + * the number of bytes read. + */ +static int +read_block(int length) +{ + char *buf; + int n, nbuf; + + buf = malloc(length); + nbuf = 0; + while (nbuf < length) { + n = read(sock, buf + nbuf, + (2048 < length - nbuf ? 2048 : length - nbuf)); + if (n <= 0) { + perror("failed reading the block\n"); + break; + } + nbuf += n; + } + free(buf); + return nbuf; +} + +/* Receive the response after sending a request. + */ +static int +receive_response(void) +{ + char *line, *end; + const char *next; + int line_len; + long content_length = -1; + int chunked = 0; + int close_connection = 0; + int req_failed = 1; + int n; + long block_len; + int status; + + /* Read header */ + while (1) { + line_len = read_line(&line); + end = line + line_len; + + if (*line == '\r' && *(line + 1) == '\n') { + free(line); + break; + } + + if (strncmp(line, "HTTP", 4) == 0) { + sscanf(line, "%*s %d %*s\r\n", &status); + req_failed = (status != 200); + } else if (isprefix(line, "content-length:", end, &next)) + content_length = strtol(next, &end, 10); + else if (isprefix(line, "encoding:", end, &next) || + isprefix(line, "transfer-encoding:", end, &next)) + chunked = (strstr(next, "chunked") != NULL); + else if (isprefix(line, "connection:", end, &next)) + close_connection = (strstr(next, "close") != NULL); + + free(line); + } + + if (debug) + fprintf(stderr, "status: %d\n", status); + + + /* Read body */ + if (content_length > 0 && !chunked) { + /* Fixed body size, read content_length bytes */ + if (debug) + fprintf(stderr, "fixed length\n"); + n = read_block(content_length); + if (debug) { + fprintf(stderr, "size of body: %d\n", (int)content_length); + fprintf(stderr, "bytes read: %d\n", n); + } + } else if (chunked) { + /* Chunked encoding, read size and bytes until no more */ + if (debug) + fprintf(stderr, "chunked encoding\n"); + while (1) { + line_len = read_line(&line); + end = line + line_len; + block_len = strtol(line, &end, 16); + if (block_len == 0) { + break; + } + n = read_block(block_len); + if (debug) { + fprintf(stderr, "size of body: %d\n", (int)block_len); + fprintf(stderr, "bytes read: %d\n", n); + } + free(line); + n = read_line(&line); + free(line); + } + n = read_line(&line); + free(line); + } else if ((content_length <= 0 && !chunked) || req_failed) { + /* No body --> stop reading. */ + if (debug) + fprintf(stderr, "no body\n"); + } else { + /* Unhandled case. */ + fprintf(stderr, "An error occured\n"); + exit(1); + } + if (debug) + fprintf(stderr, "\n"); + + return close_connection; +} + + +static int +gen_traffic(void *priv, enum shmlogtag tag, unsigned fd, + unsigned len, unsigned spec, const char *ptr) +{ + const char *end, *next; + FILE *fo; + struct request *rp; + + end = ptr + len; + + if (!(spec & VSL_S_CLIENT)) + return (0); + + if (fd >= nreq) { + struct request **newreq = req; + size_t newnreq = nreq; + + while (fd >= newnreq) + newnreq += newnreq + 1; + newreq = realloc(newreq, newnreq * sizeof *newreq); + assert(newreq != NULL); + memset(newreq + nreq, 0, (newnreq - nreq) * sizeof *newreq); + req = newreq; + nreq = newnreq; + } + if (req[fd] == NULL) { + req[fd] = calloc(sizeof *req[fd], 1); + assert(req[fd] != NULL); + } + rp = req[fd]; + + switch (tag) { + case SLT_RxRequest: + if (tag == SLT_RxRequest && (spec & VSL_S_BACKEND)) + break; + + if (rp->df_m != NULL) + rp->bogus = 1; + else + rp->df_m = trimline(ptr, end); + break; + + case SLT_RxURL: + if (tag == SLT_RxURL && (spec & VSL_S_BACKEND)) + break; + + if (rp->df_Uq != NULL) + rp->bogus = 1; + else + rp->df_Uq = trimline(ptr, end); + break; + + case SLT_RxProtocol: + if (tag == SLT_RxProtocol && (spec & VSL_S_BACKEND)) + break; + + if (rp->df_H != NULL) + rp->bogus = 1; + else + rp->df_H = trimline(ptr, end); + break; + + case SLT_RxHeader: + if (isprefix(ptr, "host:", end, &next)) + rp->df_Host = trimline(next, end); + if (isprefix(ptr, "connection:", end, &next)) + rp->df_c = trimline(next, end); + break; + + default: + break; + } + + if ((spec & VSL_S_CLIENT) && tag != SLT_ReqEnd) + return (0); + + if (!rp->bogus) { + fo = priv; + + /* If the method is supported (GET or HEAD), send the request out + * on the socket. If the socket needs reopening, reopen it first. + * When the request is sent, call the function for receiving + * the answer. + */ + if (!(strncmp(rp->df_m, "GET", 3) && strncmp(rp->df_m, "HEAD", 4))) { + if (reopen) + sock = VSS_connect(adr_info); + reopen = 0; + + if (debug) { + fprintf(fo, "%s ", rp->df_m); + fprintf(fo, "%s ", rp->df_Uq); + fprintf(fo, "%s ", rp->df_H); + fprintf(fo, "\n"); + fprintf(fo, "Host: "); + } + write(sock, rp->df_m, strlen(rp->df_m)); + write(sock, " ", 1); + write(sock, rp->df_Uq, strlen(rp->df_Uq)); + write(sock, " ", 1); + write(sock, rp->df_H, strlen(rp->df_H)); + write(sock, " ", 1); + write(sock, "\r\n", 2); + + if (strncmp(rp->df_H, "HTTP/1.0", 8)) + reopen = 1; + + write(sock, "Host: ", 6); + if (rp->df_Host) { + if (debug) + fprintf(fo, rp->df_Host); + write(sock, rp->df_Host, strlen(rp->df_Host)); + } + if (debug) + fprintf(fo, "\n"); + write(sock, "\r\n", 2); + if (rp->df_c) { + if (debug) + fprintf(fo, "Connection: %s\n", rp->df_c); + write(sock, "Connection: ", 12); + write(sock, rp->df_c, strlen(rp->df_c)); + write(sock, "\r\n", 2); + if (isequal(rp->df_c, "keep-alive", rp->df_c + strlen(rp->df_c))) + reopen = 0; + } + if (debug) + fprintf(fo, "\n"); + write(sock, "\r\n", 2); + if (!reopen) + reopen = receive_response(); + if (reopen) + close(sock); + } + } + + /* clean up */ +#define freez(x) do { if (x) free(x); x = NULL; } while (0); + freez(rp->df_H); + freez(rp->df_Host); + freez(rp->df_Uq); + freez(rp->df_m); + freez(rp->df_c); +#undef freez + rp->bogus = 0; + + return (0); +} + +/* This function is for testing only, and only sends + * the raw data from the file to the address. + * The receive function is called for each blank line. + */ +static void +send_test_request(char *file, const char *address) +{ + int fd = open(file, O_RDONLY); + char buf[2]; + char last = ' '; + adr_info = init_connection(address); + sock = VSS_connect(adr_info); + while (read(fd, buf, 1)) { + write(sock, buf, 1); + fprintf(stderr, "%s", buf); + if (*buf == '\n' && last == '\n'){ + fprintf(stderr, "receive\n"); + reopen = receive_response(); + } + last = *buf; + } + close(sock); + +} + +/*--------------------------------------------------------------------*/ + +static void +usage(void) +{ + + fprintf(stderr, "usage: varnishreplay -a address:port -r logfile [-D]\n"); + exit(1); +} + +int +main(int argc, char *argv[]) +{ + int i, c; + struct VSL_data *vd; + const char *ofn = NULL; + const char *address = NULL; + FILE *of; + + char *test_file = NULL; + + vd = VSL_New(); + debug = 0; + + while ((c = getopt(argc, argv, "a:Dr:t:")) != -1) { + i = VSL_Arg(vd, c, optarg); + if (i < 0) + exit (1); + if (i > 0) + continue; + switch (c) { + case 'a': + address = optarg; + break; + case 'D': + debug = 1; + break; + case 't': + /* This option is for testing only. The test file must contain + * a sequence of valid HTTP-requests that can be sent + * unchanged to the adress given with -a + */ + test_file = optarg; + break; + default: + if (VSL_Arg(vd, c, optarg) > 0) + break; + usage(); + } + } + + if (test_file != NULL) { + send_test_request(test_file, address); + exit(0); + } + + if (address == NULL) { + usage(); + } + + if (VSL_OpenLog(vd, NULL)) + exit(1); + + ofn = "stdout"; + of = stdout; + + adr_info = init_connection(address); + reopen = 1; + + while (VSL_Dispatch(vd, gen_traffic, of) == 0) { + if (fflush(of) != 0) { + perror(ofn); + exit(1); + } + } + + exit(0); +} + diff --git a/varnish-cache/configure.ac b/varnish-cache/configure.ac index 4edfc5aa..9af007fe 100644 --- a/varnish-cache/configure.ac +++ b/varnish-cache/configure.ac @@ -138,6 +138,7 @@ AC_CONFIG_FILES([ bin/varnishncsa/Makefile bin/varnishstat/Makefile bin/varnishtop/Makefile + bin/varnishreplay/Makefile doc/Makefile etc/Makefile include/Makefile