From 330a2a6041fc51b26ae19c795a059d888733d6da Mon Sep 17 00:00:00 2001 From: Torsten Werner Date: Sun, 4 Jul 2010 15:05:03 +0200 Subject: [PATCH] add threadpool implementation ... and its license information Signed-off-by: Torsten Werner --- COPYING-PSF | 275 +++++++++++++++++++++++++++++++++++++++++++ daklib/threadpool.py | 224 +++++++++++++++++++++++++++++++++++ 2 files changed, 499 insertions(+) create mode 100644 COPYING-PSF create mode 100644 daklib/threadpool.py diff --git a/COPYING-PSF b/COPYING-PSF new file mode 100644 index 00000000..5998dbf6 --- /dev/null +++ b/COPYING-PSF @@ -0,0 +1,275 @@ +Python 2.6.2 license + +This is the official license for the Python 2.6.2 release and it is used +for the file daklib/threadpool.py: + +A. HISTORY OF THE SOFTWARE +========================== + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations (now Zope +Corporation, see http://www.zope.com). In 2001, the Python Software +Foundation (PSF, see http://www.python.org/psf/) was formed, a +non-profit organization created specifically to own Python-related +Intellectual Property. Zope Corporation is a sponsoring member of +the PSF. + +All Python releases are Open Source (see http://www.opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases. + + Release Derived Year Owner GPL- + from compatible? (1) + + 0.9.0 thru 1.2 1991-1995 CWI yes + 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes + 1.6 1.5.2 2000 CNRI no + 2.0 1.6 2000 BeOpen.com no + 1.6.1 1.6 2001 CNRI yes (2) + 2.1 2.0+1.6.1 2001 PSF no + 2.0.1 2.0+1.6.1 2001 PSF yes + 2.1.1 2.1+2.0.1 2001 PSF yes + 2.2 2.1.1 2001 PSF yes + 2.1.2 2.1.1 2002 PSF yes + 2.1.3 2.1.2 2002 PSF yes + 2.2.1 2.2 2002 PSF yes + 2.2.2 2.2.1 2002 PSF yes + 2.2.3 2.2.2 2003 PSF yes + 2.3 2.2.2 2002-2003 PSF yes + 2.3.1 2.3 2002-2003 PSF yes + 2.3.2 2.3.1 2002-2003 PSF yes + 2.3.3 2.3.2 2002-2003 PSF yes + 2.3.4 2.3.3 2004 PSF yes + 2.3.5 2.3.4 2005 PSF yes + 2.4 2.3 2004 PSF yes + 2.4.1 2.4 2005 PSF yes + 2.4.2 2.4.1 2005 PSF yes + 2.4.3 2.4.2 2006 PSF yes + 2.5 2.4 2006 PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, + because its license has a choice of law clause. According to + CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 + is "not incompatible" with the GPL. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python +alone or in any derivative version, provided, however, that PSF's +License Agreement and PSF's notice of copyright, i.e., "Copyright (c) +2001, 2002, 2003, 2004, 2005, 2006 Python Software Foundation; All Rights +Reserved" are retained in Python alone or in any derivative version +prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the Internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the Internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + diff --git a/daklib/threadpool.py b/daklib/threadpool.py new file mode 100644 index 00000000..95a14a34 --- /dev/null +++ b/daklib/threadpool.py @@ -0,0 +1,224 @@ +import threading +from time import sleep + +# This code is a modified copy of +# http://code.activestate.com/recipes/203871-a-generic-programming-thread-pool/ +# and is licensed under the Python License. The full text of the license +# is available in the file COPYING-PSF. + +# FIXME: +# numThreads defaults to 16 in __init__ to work best on +# franck.debian.org but the default value should be specified in +# dak.conf + +# Ensure booleans exist (not needed for Python 2.2.1 or higher) +try: + True +except NameError: + False = 0 + True = not False + +class ThreadPool: + + """Flexible thread pool class. Creates a pool of threads, then + accepts tasks that will be dispatched to the next available + thread.""" + + def __init__(self, numThreads = 16): + + """Initialize the thread pool with numThreads workers.""" + + self.__threads = [] + self.__resizeLock = threading.Condition(threading.Lock()) + self.__taskLock = threading.Condition(threading.Lock()) + self.__tasks = [] + self.__isJoining = False + self.setThreadCount(numThreads) + + def setThreadCount(self, newNumThreads): + + """ External method to set the current pool size. Acquires + the resizing lock, then calls the internal version to do real + work.""" + + # Can't change the thread count if we're shutting down the pool! + if self.__isJoining: + return False + + self.__resizeLock.acquire() + try: + self.__setThreadCountNolock(newNumThreads) + finally: + self.__resizeLock.release() + return True + + def __setThreadCountNolock(self, newNumThreads): + + """Set the current pool size, spawning or terminating threads + if necessary. Internal use only; assumes the resizing lock is + held.""" + + # If we need to grow the pool, do so + while newNumThreads > len(self.__threads): + newThread = ThreadPoolThread(self) + self.__threads.append(newThread) + newThread.start() + # If we need to shrink the pool, do so + while newNumThreads < len(self.__threads): + self.__threads[0].goAway() + del self.__threads[0] + + def getThreadCount(self): + + """Return the number of threads in the pool.""" + + self.__resizeLock.acquire() + try: + return len(self.__threads) + finally: + self.__resizeLock.release() + + def queueTask(self, task, args=None, taskCallback=None): + + """Insert a task into the queue. task must be callable; + args and taskCallback can be None.""" + + if self.__isJoining == True: + return False + if not callable(task): + return False + + self.__taskLock.acquire() + try: + self.__tasks.append((task, args, taskCallback)) + return True + finally: + self.__taskLock.release() + + def getNextTask(self): + + """ Retrieve the next task from the task queue. For use + only by ThreadPoolThread objects contained in the pool.""" + + self.__taskLock.acquire() + try: + if self.__tasks == []: + return (None, None, None) + else: + return self.__tasks.pop(0) + finally: + self.__taskLock.release() + + def joinAll(self, waitForTasks = True, waitForThreads = True): + + """ Clear the task queue and terminate all pooled threads, + optionally allowing the tasks and threads to finish.""" + + # Mark the pool as joining to prevent any more task queueing + self.__isJoining = True + + # Wait for tasks to finish + if waitForTasks: + while self.__tasks != []: + sleep(.1) + + # Tell all the threads to quit + self.__resizeLock.acquire() + try: + self.__setThreadCountNolock(0) + self.__isJoining = True + + # Wait until all threads have exited + if waitForThreads: + for t in self.__threads: + t.join() + del t + + # Reset the pool for potential reuse + self.__isJoining = False + finally: + self.__resizeLock.release() + + + +class ThreadPoolThread(threading.Thread): + + """ Pooled thread class. """ + + threadSleepTime = 0.1 + + def __init__(self, pool): + + """ Initialize the thread and remember the pool. """ + + threading.Thread.__init__(self) + self.__pool = pool + self.__isDying = False + + def run(self): + + """ Until told to quit, retrieve the next task and execute + it, calling the callback if any. """ + + while self.__isDying == False: + cmd, args, callback = self.__pool.getNextTask() + # If there's nothing to do, just sleep a bit + if cmd is None: + sleep(ThreadPoolThread.threadSleepTime) + elif callback is None: + cmd(args) + else: + callback(cmd(args)) + + def goAway(self): + + """ Exit the run loop next time through.""" + + self.__isDying = True + +# Usage example +if __name__ == "__main__": + + from random import randrange + + # Sample task 1: given a start and end value, shuffle integers, + # then sort them + + def sortTask(data): + print "SortTask starting for ", data + numbers = range(data[0], data[1]) + for a in numbers: + rnd = randrange(0, len(numbers) - 1) + a, numbers[rnd] = numbers[rnd], a + print "SortTask sorting for ", data + numbers.sort() + print "SortTask done for ", data + return "Sorter ", data + + # Sample task 2: just sleep for a number of seconds. + + def waitTask(data): + print "WaitTask starting for ", data + print "WaitTask sleeping for %d seconds" % data + sleep(data) + return "Waiter", data + + # Both tasks use the same callback + + def taskCallback(data): + print "Callback called for", data + + # Create a pool with three worker threads + + pool = ThreadPool(3) + + # Insert tasks into the queue and let them run + pool.queueTask(sortTask, (1000, 100000), taskCallback) + pool.queueTask(waitTask, 5, taskCallback) + pool.queueTask(sortTask, (200, 200000), taskCallback) + pool.queueTask(waitTask, 2, taskCallback) + pool.queueTask(sortTask, (3, 30000), taskCallback) + pool.queueTask(waitTask, 7, taskCallback) + + # When all tasks are finished, allow the threads to terminate + pool.joinAll() -- 2.39.5