From 38b0ccecf49449fb6350bbe229a50eb966f4b5f5 Mon Sep 17 00:00:00 2001 From: phk Date: Thu, 19 Feb 2009 16:00:22 +0000 Subject: [PATCH] Split a couple of generally useful routines out of storage_file git-svn-id: svn+ssh://projects.linpro.no/svn/varnish/trunk@3788 d4fa192b-c00b-0410-8231-f00ffab90ce4 --- varnish-cache/bin/varnishd/Makefile.am | 1 + varnish-cache/bin/varnishd/stevedore.h | 3 + varnish-cache/bin/varnishd/stevedore_utils.c | 237 +++++++++++++++++++ varnish-cache/bin/varnishd/storage_file.c | 170 +------------ 4 files changed, 246 insertions(+), 165 deletions(-) create mode 100644 varnish-cache/bin/varnishd/stevedore_utils.c diff --git a/varnish-cache/bin/varnishd/Makefile.am b/varnish-cache/bin/varnishd/Makefile.am index 1a858527..c04105d0 100644 --- a/varnish-cache/bin/varnishd/Makefile.am +++ b/varnish-cache/bin/varnishd/Makefile.am @@ -55,6 +55,7 @@ varnishd_SOURCES = \ storage_malloc.c \ storage_synth.c \ storage_umem.c \ + stevedore_utils.c \ varnishd.c noinst_HEADERS = \ diff --git a/varnish-cache/bin/varnishd/stevedore.h b/varnish-cache/bin/varnishd/stevedore.h index c8d2905a..212c8796 100644 --- a/varnish-cache/bin/varnishd/stevedore.h +++ b/varnish-cache/bin/varnishd/stevedore.h @@ -63,5 +63,8 @@ void STV_free(struct storage *st); void STV_add(const struct stevedore *stv, int ac, char * const *av); void STV_open(void); +int STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx); +uintmax_t STV_FileSize(int fd, const char *size, unsigned *granularity, const char *ctx); + /* Synthetic Storage */ void SMS_Init(void); diff --git a/varnish-cache/bin/varnishd/stevedore_utils.c b/varnish-cache/bin/varnishd/stevedore_utils.c new file mode 100644 index 00000000..d843421b --- /dev/null +++ b/varnish-cache/bin/varnishd/stevedore_utils.c @@ -0,0 +1,237 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2008 Linpro AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id$ + * + * Utility functions for stevedores and storage modules + */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_SYS_MOUNT_H +#include +#endif + +#ifdef HAVE_SYS_STATVFS_H +#include +#endif + +#ifdef HAVE_SYS_VFS_H +#include +#endif + +#include "mgt.h" +#include "stevedore.h" + +#ifndef O_LARGEFILE +#define O_LARGEFILE 0 +#endif + +/*-------------------------------------------------------------------- + * Get a storage file. + * + * The fn argument can be an existing file, an existing directory or + * a nonexistent filename in an existing directory. + * + * If a directory is specified, the file will be anonymous (unlinked) + * + * Return: + * 0 if the file was preexisting. + * 1 if the file was created. + * 2 if the file is anonymous. + * + * Uses ARGV_ERR to exit in case of trouble. + */ + +int +STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx) +{ + int fd; + struct stat st; + char *q; + int retval = 1; + + AN(fn); + AN(fnp); + AN(fdp); + *fnp = NULL; + *fdp = -1; + + /* try to create a new file of this name */ + fd = open(fn, O_RDWR | O_CREAT | O_EXCL | O_LARGEFILE, 0600); + if (fd >= 0) { + *fdp = fd; + *fnp = fn; + return (retval); + } + + if (stat(fn, &st)) + ARGV_ERR( + "(%s) \"%s\" does not exist and could not be created\n", + ctx, fn); + + if (S_ISDIR(st.st_mode)) { + asprintf(&q, "%s/varnish.XXXXXX", fn); + XXXAN(q); + fd = mkstemp(q); + if (fd < 0) + ARGV_ERR("(%s) \"%s\" mkstemp(%s) failed (%s)\n", + ctx, fn, q, strerror(errno)); + *fnp = q; + retval = 2; + } else if (S_ISREG(st.st_mode)) { + fd = open(fn, O_RDWR | O_LARGEFILE); + if (fd < 0) + ARGV_ERR("(%s) \"%s\" could not open (%s)\n", + ctx, fn, strerror(errno)); + *fnp = fn; + retval = 0; + } else + ARGV_ERR( + "(%s) \"%s\" is neither file nor directory\n", ctx, fn); + + AZ(fstat(fd, &st)); + if (!S_ISREG(st.st_mode)) + ARGV_ERR("(%s) \"%s\" was not a file after opening\n", + ctx, fn); + + *fdp = fd; + return (retval); +} + +/*-------------------------------------------------------------------- + * Figure out how much space is in a filesystem + */ + +static uintmax_t +stv_fsspace(int fd, unsigned *bs) +{ +#if defined(HAVE_SYS_STATVFS_H) + struct statvfs fsst; + + AZ(fstatvfs(fd, &fsst)); +#elif defined(HAVE_SYS_MOUNT_H) || defined(HAVE_SYS_VFS_H) + struct statfs fsst; + + AZ(fstatfs(sc->fd, &fsst)); +#else +#error no struct statfs / struct statvfs +#endif + + /* We use units of the larger of filesystem blocksize and pagesize */ + if (*bs < fsst.f_bsize) + *bs = fsst.f_bsize; + xxxassert(*bs % fsst.f_bsize == 0); + return (fsst.f_bsize * fsst.f_bavail); +} + + +/*-------------------------------------------------------------------- + * Decide file size. + * + * If the sizespecification is empty and the file exists with non-zero + * size, use that, otherwise, interpret the specification. + * + * Handle off_t sizes and pointer width limitations. + */ + +uintmax_t +STV_FileSize(int fd, const char *size, unsigned *granularity, const char *ctx) +{ + uintmax_t l, fssize; + unsigned bs; + const char *q; + int i; + off_t o; + struct stat st; + + AZ(fstat(fd, &st)); + xxxassert(S_ISREG(st.st_mode)); + + bs = *granularity; + fssize = stv_fsspace(fd, &bs); + xxxassert(bs % *granularity == 0); + + if ((size == NULL || *size == '\0') && st.st_size != 0) { + /* + * We have no size specification, but an existing file, + * use it's existing size. + */ + l = st.st_size; + } else { + AN(size); + q = str2bytes(size, &l, fssize); + + if (q != NULL) + ARGV_ERR("(%s) size \"%s\": %s\n", size, ctx, q); + } + + /* + * This trickery wouldn't be necessary if X/Open would + * just add OFF_MAX to ... + */ + i = 0; + while(1) { + o = l; + if (o == l && o > 0) + break; + l >>= 1; + i++; + } + if (i) + fprintf(stderr, "WARNING: (%s) file size reduced" + " to %ju due to system \"off_t\" limitations\n", ctx, l); + else if (l - st.st_size > fssize) { + l = fssize * 80 / 100; + fprintf(stderr, "WARNING: (%s) file size reduced" + " to %ju (80%% of available disk space)\n", ctx, l); + } + + if (sizeof(void *) == 4 && l > INT32_MAX) { /*lint !e506 !e774 */ + fprintf(stderr, + "NB: Storage size limited to 2GB on 32 bit architecture,\n" + "NB: otherwise we could run out of address space.\n" + ); + l = INT32_MAX; + } + + /* round down to multiple of filesystem blocksize or pagesize */ + l -= (l % bs); + + *granularity = bs; + return(l); +} diff --git a/varnish-cache/bin/varnishd/storage_file.c b/varnish-cache/bin/varnishd/storage_file.c index 13dc4bde..e9fc1513 100644 --- a/varnish-cache/bin/varnishd/storage_file.c +++ b/varnish-cache/bin/varnishd/storage_file.c @@ -38,18 +38,6 @@ #include #include -#ifdef HAVE_SYS_MOUNT_H -#include -#endif - -#ifdef HAVE_SYS_STATVFS_H -#include -#endif - -#ifdef HAVE_SYS_VFS_H -#include -#endif - #include #include #include @@ -118,115 +106,15 @@ struct smf_sc { struct lock mtx; }; -/*--------------------------------------------------------------------*/ - -static uintmax_t -smf_fsspace(int fd, unsigned *bs) -{ -#if defined(HAVE_SYS_STATVFS_H) - struct statvfs fsst; - - AZ(fstatvfs(fd, &fsst)); -#elif defined(HAVE_SYS_MOUNT_H) || defined(HAVE_SYS_VFS_H) - struct statfs fsst; - - AZ(fstatfs(sc->fd, &fsst)); -#else -#error no struct statfs / struct statvfs -#endif - - /* We use units of the larger of filesystem blocksize and pagesize */ - if (*bs < fsst.f_bsize) - *bs = fsst.f_bsize; - xxxassert(*bs % fsst.f_bsize == 0); - return (fsst.f_bsize * fsst.f_bavail); -} - - /*--------------------------------------------------------------------*/ static void -smf_calcsize(struct smf_sc *sc, const char *size, int newfile) +smf_initfile(struct smf_sc *sc, const char *size) { - uintmax_t l, fssize; - unsigned bs; - const char *q; - int i; - off_t o; - struct stat st; - - AN(sc); - AZ(fstat(sc->fd, &st)); - xxxassert(S_ISREG(st.st_mode)); - - bs = sc->pagesize; - fssize = smf_fsspace(sc->fd, &bs); - xxxassert(bs % sc->pagesize == 0); - - if ((size == NULL || *size == '\0') && !newfile) { - /* - * We have no size specification, but an existing file, - * use it's existing size. - */ - l = st.st_size; - } else { - AN(size); - q = str2bytes(size, &l, fssize); - - if (q != NULL) - ARGV_ERR("(-sfile) size \"%s\": %s\n", size, q); - } - - /* - * This trickery wouldn't be necessary if X/Open would - * just add OFF_MAX to ... - */ - i = 0; - while(1) { - o = l; - if (o == l && o > 0) - break; - l >>= 1; - i++; - } - if (i) - fprintf(stderr, "WARNING: storage file size reduced" - " to %ju due to system limitations\n", l); - - if (l < st.st_size) { - AZ(ftruncate(sc->fd, (off_t)l)); - } else if (l - st.st_size > fssize) { - l = fssize * 80 / 100; - fprintf(stderr, "WARNING: storage file size reduced" - " to %ju (80%% of available disk space)\n", l); - } - - /* round down to multiple of filesystem blocksize or pagesize */ - l -= (l % bs); - - if (l < MINPAGES * (uintmax_t)sc->pagesize) - ARGV_ERR("size too small, at least %ju needed\n", - (uintmax_t)MINPAGES * sc->pagesize); - - if (sizeof(void *) == 4 && l > INT32_MAX) { /*lint !e506 !e774 */ - fprintf(stderr, - "NB: Storage size limited to 2GB on 32 bit architecture,\n" - "NB: otherwise we could run out of address space.\n" - ); - l = INT32_MAX; - l -= (l % bs); - } + sc->filesize = STV_FileSize(sc->fd, size, &sc->pagesize, "-sfile"); printf("storage_file: filename: %s size %ju MB.\n", - sc->filename, l / (1024 * 1024)); - - sc->filesize = l; -} - -static void -smf_initfile(struct smf_sc *sc, const char *size, int newfile) -{ - smf_calcsize(sc, size, newfile); + sc->filename, sc->filesize / (1024 * 1024)); AZ(ftruncate(sc->fd, (off_t)sc->filesize)); @@ -240,8 +128,6 @@ static void smf_init(struct stevedore *parent, int ac, char * const *av) { const char *size, *fn, *r; - char *q, *p; - struct stat st; struct smf_sc *sc; unsigned u; uintmax_t page_size; @@ -278,56 +164,10 @@ smf_init(struct stevedore *parent, int ac, char * const *av) parent->priv = sc; - /* try to create a new file of this name */ -#ifdef O_LARGEFILE - sc->fd = open(fn, O_RDWR | O_CREAT | O_EXCL | O_LARGEFILE, 0600); -#else - sc->fd = open(fn, O_RDWR | O_CREAT | O_EXCL, 0600); -#endif - if (sc->fd >= 0) { - sc->filename = fn; - mgt_child_inherit(sc->fd, "storage_file"); - smf_initfile(sc, size, 1); - return; - } - - /* it must exist then */ - if (stat(fn, &st)) - ARGV_ERR("(-sfile) \"%s\" " - "does not exist and could not be created\n", fn); - - /* and it should be a file or directory */ - if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode))) - ARGV_ERR("(-sfile) \"%s\" is neither file nor directory\n", fn); - - if (S_ISREG(st.st_mode)) { - sc->fd = open(fn, O_RDWR); - if (sc->fd < 0) - ARGV_ERR("(-sfile) \"%s\" could not open (%s)\n", - fn, strerror(errno)); - AZ(fstat(sc->fd, &st)); - if (!S_ISREG(st.st_mode)) - ARGV_ERR("(-sfile) \"%s\" " - "was not a file after opening\n", fn); - sc->filename = fn; - mgt_child_inherit(sc->fd, "storage_file"); - smf_initfile(sc, size, 0); - return; - } + (void)STV_GetFile(fn, &sc->fd, &sc->filename, "-sfile"); - asprintf(&q, "%s/varnish.XXXXXX", fn); - XXXAN(q); - sc->fd = mkstemp(q); - if (sc->fd < 0) - ARGV_ERR("(-sfile) \"%s\" " - "mkstemp(%s) failed (%s)\n", fn, q, strerror(errno)); - AZ(unlink(q)); - asprintf(&p, "%s (unlinked)", q); - XXXAN(p); - sc->filename = p; - free(q); - smf_initfile(sc, size, 1); mgt_child_inherit(sc->fd, "storage_file"); + smf_initfile(sc, size); } /*-------------------------------------------------------------------- -- 2.39.5