From: znek Date: Fri, 21 May 2004 09:17:32 +0000 (+0000) Subject: Added helge's tests X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9e9903296832975ae9bd672e3bc14afefd99cc87;p=scalable-opengroupware.org Added helge's tests git-svn-id: http://svn.opengroupware.org/SOGo/trunk@2 d1b88da0-ebda-0310-925b-ed51d893ca5b --- diff --git a/db/tests/helge/NOTES b/db/tests/helge/NOTES new file mode 100644 index 00000000..294ab7e3 --- /dev/null +++ b/db/tests/helge/NOTES @@ -0,0 +1,150 @@ +# $Id$ + +Issues +- folder generation? + - addition der item-generations geht nicht, weil auch records geloescht + werden koennen +- separate tabelle fuer "proposed" appointments? + - damit wir die Apts nicht schicken muessen und die erst im scheduler + angezeigt werden, wenn sie aktuell sind + +Move Databases +============== + +HOST="localhost" +USER="agenor" + +DB="blah2" +DB="agenor%i" % ( i, ) + +DB="agenortabledb" +NEWTABLE="agenor_tab_%i" % ( i, ) + +DB="agenor_fldinfodb" + +DB="agenor_testhugeperf" + + +Schemas +======= + +CREATE TABLE SOGo_folder_info ( + c_foldername VARCHAR(255) NOT NULL, + c_tablename VARCHAR(255) NOT NULL, + c_dbname VARCHAR(255) NOT NULL +); +INSERT INTO SOGo_folder_info (c_foldername,c_tablename,c_dbname) + VALUES ('hh calendar', 'agenor_tab_105', 'agenortabledb'); + +CREATE UNIQUE INDEX SOGo_folder_info_idx + ON SOGo_folder_info + USING BTREE(c_foldername); +(ca 5s) + +--- + CREATE TABLE SOGo_huge_ical ( + c_pkey INT PRIMARY KEY, + c_data VARCHAR(32000) NOT NULL + ); + CREATE TABLE SOGo_huge_quick ( + c_pkey INT PRIMARY KEY, + c_sourceid VARCHAR(255) NOT NULL, + c_startdate INT NOT NULL, + c_enddate INT NOT NULL, + c_title VARCHAR(255) NOT NULL, + c_attendees VARCHAR(4096) NOT NULL, + c_isallday INT NOT NULL, + c_sequenceid INT NOT NULL, + c_generation INT NOT NULL + ); +CREATE INDEX SOGo_huge_quick_idx + ON SOGo_huge_quick + USING BTREE(c_startdate); +(ca 15s on 1.000.000) +--- + + +Performance +=========== +agenor_fldinfodb: + 10000 Folder Info Entries, kein Index: 71s + 100000 Folder Info Entries: 12:09m, 729s, 137 inserts/s + 5992424=>6001088, diff 8664KB data size (von ca 16000 auf 110000) + ~94 byte per row (raw: ~12+14+13=39 byte) + 110001-250000 Folder Info Entries: 15:59m, 959s, 145 inserts/s + 6001088-6014316, diff 13228KB data size (~96 byte per row) + + ohne Index, via Python: + COUNT(*) => 360ms + c_tablename, c_dbname | * => 1128ms + c_tablename, c_dbname | c_foldername='privcal_99827' => 345ms + mit Index, via Python + COUNT(*) => 350ms + c_tablename, c_dbname | * => 1124ms + c_tablename, c_dbname | c_foldername='privcal_99827' => 18,5,5ms + +agenor_testhugeperf: + 1.000.000 entries, kein Index: + 10000=79.37s, 20000=162s, 30000=245s,340000=2831s,790000=6670s + (~120 rows per sec) = > ca 2h fuer 1.000.000, 20h fuer 10.000.000 + 30000=65MB => ~2KB per Record => ~2GB fuer 1.000.000 + 220000=440MB,810000=1.55GB,1.000.000= + ~1.92GB x 20 = 40GB + duration:~2:50h + ohne Index, via Python: + COUNT(*) => 20.8s + pkey,sourceid,start,end | c_start>x&c_end 13.4s + c_sourceid | * => 10.5s + mit Index, via Python + COUNT(*) => 9.7s,3s,2.5s + pkey,sourceid,start,end | c_start>x&c_end 86ms,59ms,17ms,17ms + c_sourceid | * => 9.3s,4.8s,4.8s + vacuum analyze: 30s? + ohne fsync, 2 gleichzeitig + - 1108 rows pro sec in einer connection! (1.000.000=ca 15min) + 53:59m + 53:52m + ~8 gleichzeitig + ~20:00:- + +http://www.varlena.com/varlena/GeneralBits/Tidbits/perf.html +http://www.varlena.com/varlena/GeneralBits/Tidbits/annotated_conf_e.html +http://www.lyris.com/lm_help/7.8/tuning_postgresql.html +http://www.linuxjournal.com/article.php?sid=4791 +http://www.argudo.org/postgresql/soft-tuning.php#s2 + +PostgreSQL Index +================ +http://www.postgresql.org/docs/current/static/sql-createindex.html +http://postgis.refractions.net/docs/x511.html + +"After building an index, it is important to force PostgreSQL to collect table statistics, which are used to optimize query plans: VACUUM ANALIZE;" + +PostgreSQL provides the index methods B-tree, R-tree, hash, and GiST +- only the B-tree and GiST index methods support multicolumn indexes + +---snip--- +An index field can be an expression computed from the values of one or more +columns of the table row. This feature can be used to obtain fast access to +data based on some transformation of the basic data. For example, an index +computed on upper(col) would allow the clause WHERE upper(col) = 'JIM' to use +an index. +---snap--- + +---snip--- +Note: Because of the limited utility of hash indexes, a B-tree index should +generally be preferred over a hash index. We do not have sufficient evidence +that hash indexes are actually faster than B-trees even for = comparisons. +Moreover, hash indexes require coarser locks; see Section 9.7. +---snap-- + +PostgreSQL Arrays +================= +---snip--- +It all depends, I have found array operations to be slow. So if you have just +a few elements, like less than 10 then arrays are ok, but with more e.g. 30+ +elements, as in my case, imho the whole exercise is not really feasable. I am +going to re-design the schema to get rid of the arrays as soon as I have a +moment. Also the code around that part of PostgreSQL has not been visited for +a fair while and needs some polishing up. I'd avoid them. +---snap--- diff --git a/db/tests/helge/PyGreSQL-samples/advanced.py b/db/tests/helge/PyGreSQL-samples/advanced.py new file mode 100755 index 00000000..41a5bc45 --- /dev/null +++ b/db/tests/helge/PyGreSQL-samples/advanced.py @@ -0,0 +1,198 @@ +#! /usr/bin/env python +# advanced.py - demo of advanced features of PostGres. Some may not be ANSI. +# inspired from the Postgres tutorial +# adapted to Python 1995 by Pascal Andre + +print """ +__________________________________________________________________ +MODULE ADVANCED.PY : ADVANCED POSTGRES SQL COMMANDS TUTORIAL + +This module is designed for being imported from python prompt + +In order to run the samples included here, first create a connection +using : cnx = advanced.DB(...) + +The "..." should be replaced with whatever arguments you need to open an +existing database. Usually all you need is the name of the database and, +in fact, if it is the same as your login name, you can leave it empty. + +then start the demo with: advanced.demo(cnx) +__________________________________________________________________ +""" + +from pg import DB +import sys + +# waits for a key +def wait_key(): + print "Press " + sys.stdin.read(1) + +# inheritance features +def inherit_demo(pgcnx): + print "-----------------------------" + print "-- Inheritance:" + print "-- a table can inherit from zero or more tables. A query" + print "-- can reference either all rows of a table or all rows " + print "-- of a table plus all of its descendants." + print "-----------------------------" + print + print "-- For example, the capitals table inherits from cities table." + print "-- (It inherits all data fields from cities.)" + print + print "CREATE TABLE cities (" + print " name text," + print " population float8," + print " altitude int" + print ")" + print + print "CREATE TABLE capitals (" + print " state varchar(2)" + print ") INHERITS (cities)" + pgcnx.query("""CREATE TABLE cities ( + name text, + population float8, + altitude int)""") + pgcnx.query("""CREATE TABLE capitals ( + state varchar(2)) INHERITS (cities)""") + wait_key() + print + print "-- now, let's populate the tables" + print + print "INSERT INTO cities VALUES ('San Francisco', 7.24E+5, 63)" + print "INSERT INTO cities VALUES ('Las Vegas', 2.583E+5, 2174)" + print "INSERT INTO cities VALUES ('Mariposa', 1200, 1953)" + print + print "INSERT INTO capitals VALUES ('Sacramento', 3.694E+5, 30, 'CA')" + print "INSERT INTO capitals VALUES ('Madison', 1.913E+5, 845, 'WI')" + print + pgcnx.query("INSERT INTO cities VALUES ('San Francisco', 7.24E+5, 63)") + pgcnx.query("INSERT INTO cities VALUES ('Las Vegas', 2.583E+5, 2174)") + pgcnx.query("INSERT INTO cities VALUES ('Mariposa', 1200, 1953)") + pgcnx.query("INSERT INTO capitals VALUES ('Sacramento',3.694E+5,30,'CA')") + pgcnx.query("INSERT INTO capitals VALUES ('Madison', 1.913E+5, 845, 'WI')") + print + print "SELECT * FROM cities" + print pgcnx.query("SELECT * FROM cities") + print "SELECT * FROM capitals" + print pgcnx.query("SELECT * FROM capitals") + print + print "-- like before, a regular query references rows of the base" + print "-- table only" + print + print "SELECT name, altitude" + print "FROM cities" + print "WHERE altitude > 500;" + print pgcnx.query("""SELECT name, altitude + FROM cities + WHERE altitude > 500""") + print + print "-- on the other hand, you can find all cities, including " + print "-- capitals, that are located at an altitude of 500 'ft " + print "-- or higher by:" + print + print "SELECT c.name, c.altitude" + print "FROM cities* c" + print "WHERE c.altitude > 500" + print pgcnx.query("""SELECT c.name, c.altitude + FROM cities* c + WHERE c.altitude > 500""") + +# arrays attributes +def array_demo(pgcnx): + print "----------------------" + print "-- Arrays:" + print "-- attributes can be arrays of base types or user-defined " + print "-- types" + print "----------------------" + print + print "CREATE TABLE sal_emp (" + print " name text," + print " pay_by_quarter int4[]," + print " pay_by_extra_quarter int8[]," + print " schedule text[][]" + print ")" + pgcnx.query("""CREATE TABLE sal_emp ( + name text, + pay_by_quarter int4[], + pay_by_extra_quarter int8[], + schedule text[][])""") + wait_key() + print + print "-- insert instances with array attributes. " + print " Note the use of braces" + print + print "INSERT INTO sal_emp VALUES (" + print " 'Bill'," + print " '{10000,10000,10000,10000}'," + print " '{9223372036854775800,9223372036854775800,9223372036854775800}'," + print " '{{\"meeting\", \"lunch\"}, {}}')" + print + print "INSERT INTO sal_emp VALUES (" + print " 'Carol'," + print " '{20000,25000,25000,25000}'," + print " '{9223372036854775807,9223372036854775807,9223372036854775807}'," + print " '{{\"talk\", \"consult\"}, {\"meeting\"}}')" + print + pgcnx.query("""INSERT INTO sal_emp VALUES ( + 'Bill', '{10000,10000,10000,10000}', + '{9223372036854775800,9223372036854775800,9223372036854775800}', + '{{\"meeting\", \"lunch\"}, {}}')""") + pgcnx.query("""INSERT INTO sal_emp VALUES ( + 'Carol', '{20000,25000,25000,25000}', + '{9223372036854775807,9223372036854775807,9223372036854775807}', + '{{\"talk\", \"consult\"}, {\"meeting\"}}')""") + wait_key() + print + print "----------------------" + print "-- queries on array attributes" + print "----------------------" + print + print "SELECT name FROM sal_emp WHERE" + print " sal_emp.pay_by_quarter[1] <> sal_emp.pay_by_quarter[2]" + print + print pgcnx.query("""SELECT name FROM sal_emp WHERE + sal_emp.pay_by_quarter[1] <> sal_emp.pay_by_quarter[2]""") + print + print pgcnx.query("""SELECT name FROM sal_emp WHERE + sal_emp.pay_by_extra_quarter[1] <> sal_emp.pay_by_extra_quarter[2]""") + print + print "-- retrieve third quarter pay of all employees" + print + print "SELECT sal_emp.pay_by_quarter[3] FROM sal_emp" + print + print pgcnx.query("SELECT sal_emp.pay_by_quarter[3] FROM sal_emp") + print + print "-- retrieve third quarter extra pay of all employees" + print + print "SELECT sal_emp.pay_by_extra_quarter[3] FROM sal_emp" + print pgcnx.query("SELECT sal_emp.pay_by_extra_quarter[3] FROM sal_emp") + print + print "-- retrieve first two quarters of extra quarter pay of all employees" + print + print "SELECT sal_emp.pay_by_extra_quarter[1:2] FROM sal_emp" + print + print pgcnx.query("SELECT sal_emp.pay_by_extra_quarter[1:2] FROM sal_emp") + print + print "-- select subarrays" + print + print "SELECT sal_emp.schedule[1:2][1:1] FROM sal_emp WHERE" + print " sal_emp.name = 'Bill'" + print pgcnx.query("SELECT sal_emp.schedule[1:2][1:1] FROM sal_emp WHERE " \ + "sal_emp.name = 'Bill'") + +# base cleanup +def demo_cleanup(pgcnx): + print "-- clean up (you must remove the children first)" + print "DROP TABLE sal_emp" + print "DROP TABLE capitals" + print "DROP TABLE cities;" + pgcnx.query("DROP TABLE sal_emp") + pgcnx.query("DROP TABLE capitals") + pgcnx.query("DROP TABLE cities") + +# main demo function +def demo(pgcnx): + inherit_demo(pgcnx) + array_demo(pgcnx) + demo_cleanup(pgcnx) diff --git a/db/tests/helge/PyGreSQL-samples/basics.py b/db/tests/helge/PyGreSQL-samples/basics.py new file mode 100755 index 00000000..98a7f86a --- /dev/null +++ b/db/tests/helge/PyGreSQL-samples/basics.py @@ -0,0 +1,296 @@ +#! /usr/bin/env python +# basics.py - basic SQL commands tutorial +# inspired from the Postgres95 tutorial +# adapted to Python 1995 by Pascal ANDRE + +print """ +__________________________________________________________________ +MODULE BASICS.PY : BASIC POSTGRES SQL COMMANDS TUTORIAL + +This module is designed for being imported from python prompt + +In order to run the samples included here, first create a connection +using : cnx = basics.DB(...) + +The "..." should be replaced with whatever arguments you need to open an +existing database. Usually all you need is the name of the database and, +in fact, if it is the same as your login name, you can leave it empty. + +then start the demo with: basics.demo(cnx) +__________________________________________________________________ +""" + +from pg import DB +import sys + +# waits for a key +def wait_key(): + print "Press " + sys.stdin.read(1) + +# table creation commands +def create_table(pgcnx): + print "-----------------------------" + print "-- Creating a table:" + print "-- a CREATE TABLE is used to create base tables. POSTGRES" + print "-- SQL has its own set of built-in types. (Note that" + print "-- keywords are case-insensitive but identifiers are " + print "-- case-sensitive.)" + print "-----------------------------" + print + print "Sending query :" + print "CREATE TABLE weather (" + print " city varchar(80)," + print " temp_lo int," + print " temp_hi int," + print " prcp float8," + print " date date" + print ")" + pgcnx.query("""CREATE TABLE weather (city varchar(80), temp_lo int, + temp_hi int, prcp float8, date date)""") + print + print "Sending query :" + print "CREATE TABLE cities (" + print " name varchar(80)," + print " location point" + print ")" + pgcnx.query("""CREATE TABLE cities ( + name varchar(80), + location point)""") + +# data insertion commands +def insert_data(pgcnx): + print "-----------------------------" + print "-- Inserting data:" + print "-- an INSERT statement is used to insert a new row into" + print "-- a table. There are several ways you can specify what" + print "-- columns the data should go to." + print "-----------------------------" + print + print "-- 1. the simplest case is when the list of value correspond to" + print "-- the order of the columns specified in CREATE TABLE." + print + print "Sending query :" + print "INSERT INTO weather " + print " VALUES ('San Francisco', 46, 50, 0.25, '11/27/1994')" + pgcnx.query("""INSERT INTO weather + VALUES ('San Francisco', 46, 50, 0.25, '11/27/1994')""") + print + print "Sending query :" + print "INSERT INTO cities " + print " VALUES ('San Francisco', '(-194.0, 53.0)')" + pgcnx.query("""INSERT INTO cities + VALUES ('San Francisco', '(-194.0, 53.0)')""") + print + wait_key() + print "-- 2. you can also specify what column the values correspond " + print " to. (The columns can be specified in any order. You may " + print " also omit any number of columns. eg. unknown precipitation" + print " below)" + print "Sending query :" + print "INSERT INTO weather (city, temp_lo, temp_hi, prcp, date)" + print " VALUES ('San Francisco', 43, 57, 0.0, '11/29/1994')" + pgcnx.query("INSERT INTO weather (date, city, temp_hi, temp_lo)" \ + "VALUES ('11/29/1994', 'Hayward', 54, 37)") + +# direct selection commands +def select_data1(pgcnx): + print "-----------------------------" + print "-- Retrieving data:" + print "-- a SELECT statement is used for retrieving data. The " + print "-- basic syntax is:" + print "-- SELECT columns FROM tables WHERE predicates" + print "-----------------------------" + print + print "-- a simple one would be the query:" + print "SELECT * FROM weather" + print + print "The result is :" + q = pgcnx.query("SELECT * FROM weather") + print q + print + print "-- you may also specify expressions in the target list (the " + print "-- 'AS column' specifies the column name of the result. It is " + print "-- optional.)" + print "The query :" + print " SELECT city, (temp_hi+temp_lo)/2 AS temp_avg, date " + print " FROM weather" + print "Gives :" + print pgcnx.query("""SELECT city, (temp_hi+temp_lo)/2 + AS temp_avg, date FROM weather""") + print + print "-- if you want to retrieve rows that satisfy certain condition" + print "-- (ie. a restriction), specify the condition in WHERE. The " + print "-- following retrieves the weather of San Francisco on rainy " + print "-- days." + print "SELECT *" + print "FROM weather" + print "WHERE city = 'San Francisco' " + print " and prcp > 0.0" + print pgcnx.query("""SELECT * FROM weather WHERE city = 'San Francisco' + AND prcp > 0.0""") + print + print "-- here is a more complicated one. Duplicates are removed when " + print "-- DISTINCT is specified. ORDER BY specifies the column to sort" + print "-- on. (Just to make sure the following won't confuse you, " + print "-- DISTINCT and ORDER BY can be used separately.)" + print "SELECT DISTINCT city" + print "FROM weather" + print "ORDER BY city;" + print pgcnx.query("SELECT DISTINCT city FROM weather ORDER BY city") + +# selection to a temporary table +def select_data2(pgcnx): + print "-----------------------------" + print "-- Retrieving data into other classes:" + print "-- a SELECT ... INTO statement can be used to retrieve " + print "-- data into another class." + print "-----------------------------" + print + print "The query :" + print "SELECT * INTO TABLE temptab " + print "FROM weather" + print "WHERE city = 'San Francisco' " + print " and prcp > 0.0" + pgcnx.query("""SELECT * INTO TABLE temptab FROM weather + WHERE city = 'San Francisco' and prcp > 0.0""") + print "Fills the table temptab, that can be listed with :" + print "SELECT * from temptab" + print pgcnx.query("SELECT * from temptab") + +# aggregate creation commands +def create_aggregate(pgcnx): + print "-----------------------------" + print "-- Aggregates" + print "-----------------------------" + print + print "Let's consider the query :" + print "SELECT max(temp_lo)" + print "FROM weather;" + print pgcnx.query("SELECT max(temp_lo) FROM weather") + print + print "-- Aggregate with GROUP BY" + print "SELECT city, max(temp_lo)" + print "FROM weather " + print "GROUP BY city;" + print pgcnx.query( """SELECT city, max(temp_lo) + FROM weather GROUP BY city""") + +# table join commands +def join_table(pgcnx): + print "-----------------------------" + print "-- Joining tables:" + print "-- queries can access multiple tables at once or access" + print "-- the same table in such a way that multiple instances" + print "-- of the table are being processed at the same time." + print "-----------------------------" + print + print "-- suppose we want to find all the records that are in the " + print "-- temperature range of other records. W1 and W2 are aliases " + print "--for weather." + print + print "SELECT W1.city, W1.temp_lo, W1.temp_hi, " + print " W2.city, W2.temp_lo, W2.temp_hi" + print "FROM weather W1, weather W2" + print "WHERE W1.temp_lo < W2.temp_lo " + print " and W1.temp_hi > W2.temp_hi" + print + print pgcnx.query("""SELECT W1.city, W1.temp_lo, W1.temp_hi, + W2.city, W2.temp_lo, W2.temp_hi FROM weather W1, weather W2 + WHERE W1.temp_lo < W2.temp_lo and W1.temp_hi > W2.temp_hi""") + print + print "-- let's join two tables. The following joins the weather table" + print "-- and the cities table." + print + print "SELECT city, location, prcp, date" + print "FROM weather, cities" + print "WHERE name = city" + print + print pgcnx.query("""SELECT city, location, prcp, date FROM weather, cities + WHERE name = city""") + print + print "-- since the column names are all different, we don't have to " + print "-- specify the table name. If you want to be clear, you can do " + print "-- the following. They give identical results, of course." + print + print "SELECT w.city, c.location, w.prcp, w.date" + print "FROM weather w, cities c" + print "WHERE c.name = w.city;" + print + print pgcnx.query("""SELECT w.city, c.location, w.prcp, w.date + FROM weather w, cities c WHERE c.name = w.city""") + +# data updating commands +def update_data(pgcnx): + print "-----------------------------" + print "-- Updating data:" + print "-- an UPDATE statement is used for updating data. " + print "-----------------------------" + print + print "-- suppose you discover the temperature readings are all off by" + print "-- 2 degrees as of Nov 28, you may update the data as follow:" + print + print "UPDATE weather" + print " SET temp_hi = temp_hi - 2, temp_lo = temp_lo - 2" + print " WHERE date > '11/28/1994'" + print + pgcnx.query("""UPDATE weather + SET temp_hi = temp_hi - 2, temp_lo = temp_lo - 2 + WHERE date > '11/28/1994'""") + print + print "SELECT * from weather" + print pgcnx.query("SELECT * from weather") + +# data deletion commands +def delete_data(pgcnx): + print "-----------------------------" + print "-- Deleting data:" + print "-- a DELETE statement is used for deleting rows from a " + print "-- table." + print "-----------------------------" + print + print "-- suppose you are no longer interested in the weather of " + print "-- Hayward, you can do the following to delete those rows from" + print "-- the table" + print + print "DELETE FROM weather WHERE city = 'Hayward'" + pgcnx.query("DELETE FROM weather WHERE city = 'Hayward'") + print + print "SELECT * from weather" + print + print pgcnx.query("SELECT * from weather") + print + print "-- you can also delete all the rows in a table by doing the " + print "-- following. (This is different from DROP TABLE which removes " + print "-- the table in addition to the removing the rows.)" + print + print "DELETE FROM weather" + pgcnx.query("DELETE FROM weather") + print + print "SELECT * from weather" + print pgcnx.query("SELECT * from weather") + +# table removal commands +def remove_table(pgcnx): + print "-----------------------------" + print "-- Removing the tables:" + print "-- DROP TABLE is used to remove tables. After you have" + print "-- done this, you can no longer use those tables." + print "-----------------------------" + print + print "DROP TABLE weather, cities, temptab" + pgcnx.query("DROP TABLE weather, cities, temptab") + +# main demo function +def demo(pgcnx): + create_table(pgcnx) + wait_key() + insert_data(pgcnx) + wait_key() + select_data1(pgcnx) + select_data2(pgcnx) + create_aggregate(pgcnx) + join_table(pgcnx) + update_data(pgcnx) + delete_data(pgcnx) + remove_table(pgcnx) diff --git a/db/tests/helge/PyGreSQL-samples/func.py b/db/tests/helge/PyGreSQL-samples/func.py new file mode 100755 index 00000000..af2b412b --- /dev/null +++ b/db/tests/helge/PyGreSQL-samples/func.py @@ -0,0 +1,205 @@ +# func.py - demonstrate the use of SQL functions +# inspired from the PostgreSQL tutorial +# adapted to Python 1995 by Pascal ANDRE + +print """ +__________________________________________________________________ +MODULE FUNC.PY : SQL FUNCTION DEFINITION TUTORIAL + +This module is designed for being imported from python prompt + +In order to run the samples included here, first create a connection +using : cnx = func.DB(...) + +The "..." should be replaced with whatever arguments you need to open an +existing database. Usually all you need is the name of the database and, +in fact, if it is the same as your login name, you can leave it empty. + +then start the demo with: func.demo(cnx) +__________________________________________________________________ +""" + +from pg import DB +import sys + +# waits for a key +def wait_key(): + print "Press " + sys.stdin.read(1) + +# basic functions declaration +def base_func(pgcnx): + print "-----------------------------" + print "-- Creating SQL Functions on Base Types" + print "-- a CREATE FUNCTION statement lets you create a new " + print "-- function that can be used in expressions (in SELECT, " + print "-- INSERT, etc.). We will start with functions that " + print "-- return values of base types." + print "-----------------------------" + print + print "--" + print "-- let's create a simple SQL function that takes no arguments" + print "-- and returns 1" + print + print "CREATE FUNCTION one() RETURNS int4" + print " AS 'SELECT 1 as ONE' LANGUAGE 'sql'" + pgcnx.query("""CREATE FUNCTION one() RETURNS int4 + AS 'SELECT 1 as ONE' LANGUAGE 'sql'""") + wait_key() + print + print "--" + print "-- functions can be used in any expressions (eg. in the target" + print "-- list or qualifications)" + print + print "SELECT one() AS answer" + print pgcnx.query("SELECT one() AS answer") + print + print "--" + print "-- here's how you create a function that takes arguments. The" + print "-- following function returns the sum of its two arguments:" + print + print "CREATE FUNCTION add_em(int4, int4) RETURNS int4" + print " AS 'SELECT $1 + $2' LANGUAGE 'sql'" + pgcnx.query("""CREATE FUNCTION add_em(int4, int4) RETURNS int4 + AS 'SELECT $1 + $2' LANGUAGE 'sql'""") + print + print "SELECT add_em(1, 2) AS answer" + print pgcnx.query("SELECT add_em(1, 2) AS answer") + +# functions on composite types +def comp_func(pgcnx): + print "-----------------------------" + print "-- Creating SQL Functions on Composite Types" + print "-- it is also possible to create functions that return" + print "-- values of composite types." + print "-----------------------------" + print + print "-- before we create more sophisticated functions, let's " + print "-- populate an EMP table" + print + print "CREATE TABLE EMP (" + print " name text," + print " salary int4," + print " age int4," + print " dept varchar(16)" + print ")" + pgcnx.query("""CREATE TABLE EMP ( + name text, + salary int4, + age int4, + dept varchar(16))""") + print + print "INSERT INTO EMP VALUES ('Sam', 1200, 16, 'toy')" + print "INSERT INTO EMP VALUES ('Claire', 5000, 32, 'shoe')" + print "INSERT INTO EMP VALUES ('Andy', -1000, 2, 'candy')" + print "INSERT INTO EMP VALUES ('Bill', 4200, 36, 'shoe')" + print "INSERT INTO EMP VALUES ('Ginger', 4800, 30, 'candy')" + pgcnx.query("INSERT INTO EMP VALUES ('Sam', 1200, 16, 'toy')") + pgcnx.query("INSERT INTO EMP VALUES ('Claire', 5000, 32, 'shoe')") + pgcnx.query("INSERT INTO EMP VALUES ('Andy', -1000, 2, 'candy')") + pgcnx.query("INSERT INTO EMP VALUES ('Bill', 4200, 36, 'shoe')") + pgcnx.query("INSERT INTO EMP VALUES ('Ginger', 4800, 30, 'candy')") + wait_key() + print + print "-- the argument of a function can also be a tuple. For " + print "-- instance, double_salary takes a tuple of the EMP table" + print + print "CREATE FUNCTION double_salary(EMP) RETURNS int4" + print " AS 'SELECT $1.salary * 2 AS salary' LANGUAGE 'sql'" + pgcnx.query("""CREATE FUNCTION double_salary(EMP) RETURNS int4 + AS 'SELECT $1.salary * 2 AS salary' LANGUAGE 'sql'""") + print + print "SELECT name, double_salary(EMP) AS dream" + print "FROM EMP" + print "WHERE EMP.dept = 'toy'" + print pgcnx.query("""SELECT name, double_salary(EMP) AS dream + FROM EMP WHERE EMP.dept = 'toy'""") + print + print "-- the return value of a function can also be a tuple. However," + print "-- make sure that the expressions in the target list is in the " + print "-- same order as the columns of EMP." + print + print "CREATE FUNCTION new_emp() RETURNS EMP" + print " AS 'SELECT \'None\'::text AS name," + print " 1000 AS salary," + print " 25 AS age," + print " \'none\'::varchar(16) AS dept'" + print " LANGUAGE 'sql'" + pgcnx.query("""CREATE FUNCTION new_emp() RETURNS EMP + AS 'SELECT \\\'None\\\'::text AS name, + 1000 AS salary, + 25 AS age, + \\\'none\\\'::varchar(16) AS dept' + LANGUAGE 'sql'""") + wait_key() + print + print "-- you can then project a column out of resulting the tuple by" + print "-- using the \"function notation\" for projection columns. " + print "-- (ie. bar(foo) is equivalent to foo.bar) Note that we don't" + print "-- support new_emp().name at this moment." + print + print "SELECT name(new_emp()) AS nobody" + print pgcnx.query("SELECT name(new_emp()) AS nobody") + print + print "-- let's try one more function that returns tuples" + print "CREATE FUNCTION high_pay() RETURNS setof EMP" + print " AS 'SELECT * FROM EMP where salary > 1500'" + print " LANGUAGE 'sql'" + pgcnx.query("""CREATE FUNCTION high_pay() RETURNS setof EMP + AS 'SELECT * FROM EMP where salary > 1500' + LANGUAGE 'sql'""") + print + print "SELECT name(high_pay()) AS overpaid" + print pgcnx.query("SELECT name(high_pay()) AS overpaid") + +# function with multiple SQL commands +def mult_func(pgcnx): + print "-----------------------------" + print "-- Creating SQL Functions with multiple SQL statements" + print "-- you can also create functions that do more than just a" + print "-- SELECT." + print "-----------------------------" + print + print "-- you may have noticed that Andy has a negative salary. We'll" + print "-- create a function that removes employees with negative " + print "-- salaries." + print + print "SELECT * FROM EMP" + print pgcnx.query("SELECT * FROM EMP") + print + print "CREATE FUNCTION clean_EMP () RETURNS int4" + print " AS 'DELETE FROM EMP WHERE EMP.salary <= 0" + print " SELECT 1 AS ignore_this'" + print " LANGUAGE 'sql'" + pgcnx.query("CREATE FUNCTION clean_EMP () RETURNS int4 AS 'DELETE FROM EMP WHERE EMP.salary <= 0; SELECT 1 AS ignore_this' LANGUAGE 'sql'") + print + print "SELECT clean_EMP()" + print pgcnx.query("SELECT clean_EMP()") + print + print "SELECT * FROM EMP" + print pgcnx.query("SELECT * FROM EMP") + +# base cleanup +def demo_cleanup(pgcnx): + print "-- remove functions that were created in this file" + print + print "DROP FUNCTION clean_EMP()" + print "DROP FUNCTION high_pay()" + print "DROP FUNCTION new_emp()" + print "DROP FUNCTION add_em(int4, int4)" + print "DROP FUNCTION one()" + print + print "DROP TABLE EMP CASCADE" + pgcnx.query("DROP FUNCTION clean_EMP()") + pgcnx.query("DROP FUNCTION high_pay()") + pgcnx.query("DROP FUNCTION new_emp()") + pgcnx.query("DROP FUNCTION add_em(int4, int4)") + pgcnx.query("DROP FUNCTION one()") + pgcnx.query("DROP TABLE EMP CASCADE") + +# main demo function +def demo(pgcnx): + base_func(pgcnx) + comp_func(pgcnx) + mult_func(pgcnx) + demo_cleanup(pgcnx) diff --git a/db/tests/helge/PyGreSQL-samples/syscat.py b/db/tests/helge/PyGreSQL-samples/syscat.py new file mode 100755 index 00000000..1ab1d584 --- /dev/null +++ b/db/tests/helge/PyGreSQL-samples/syscat.py @@ -0,0 +1,149 @@ +# syscat.py - parses some system catalogs +# inspired from the PostgreSQL tutorial +# adapted to Python 1995 by Pascal ANDRE + +print """ +__________________________________________________________________ +MODULE SYSCAT.PY : PARSES SOME POSTGRESQL SYSTEM CATALOGS + +This module is designed for being imported from python prompt + +In order to run the samples included here, first create a connection +using : cnx = syscat.DB(...) + +The "..." should be replaced with whatever arguments you need to open an +existing database. Usually all you need is the name of the database and, +in fact, if it is the same as your login name, you can leave it empty. + +then start the demo with: syscat.demo(cnx) + +Some results may be empty, depending on your base status." + +__________________________________________________________________ +""" + +from pg import DB +import sys + +# waits for a key +def wait_key(): + print "Press " + sys.stdin.read(1) + +# lists all simple indices +def list_simple_ind(pgcnx): + result = pgcnx.query("""SELECT bc.relname AS class_name, + ic.relname AS index_name, a.attname + FROM pg_class bc, pg_class ic, pg_index i, pg_attribute a + WHERE i.indrelid = bc.oid AND i.indexrelid = bc.oid + AND i.indkey[0] = a.attnum AND a.attrelid = bc.oid + AND i.indproc = '0'::oid AND a.attisdropped = 'f' + ORDER BY class_name, index_name, attname""") + return result + +# list all user defined attributes and their type in user-defined classes +def list_all_attr(pgcnx): + result = pgcnx.query("""SELECT c.relname, a.attname, t.typname + FROM pg_class c, pg_attribute a, pg_type t + WHERE c.relkind = 'r' and c.relname !~ '^pg_' + AND c.relname !~ '^Inv' and a.attnum > 0 + AND a.attrelid = c.oid and a.atttypid = t.oid + AND a.attisdropped = 'f' + ORDER BY relname, attname""") + return result + +# list all user defined base type +def list_user_base_type(pgcnx): + result = pgcnx.query("""SELECT u.usename, t.typname + FROM pg_type t, pg_user u + WHERE u.usesysid = int2in(int4out(t.typowner)) + AND t.typrelid = '0'::oid and t.typelem = '0'::oid + AND u.usename <> 'postgres' order by usename, typname""") + return result + +# list all right-unary operators +def list_right_unary_operator(pgcnx): + result = pgcnx.query("""SELECT o.oprname AS right_unary, + lt.typname AS operand, result.typname AS return_type + FROM pg_operator o, pg_type lt, pg_type result + WHERE o.oprkind='r' and o.oprleft = lt.oid + AND o.oprresult = result.oid + ORDER BY operand""") + return result + +# list all left-unary operators +def list_left_unary_operator(pgcnx): + result = pgcnx.query("""SELECT o.oprname AS left_unary, + rt.typname AS operand, result.typname AS return_type + FROM pg_operator o, pg_type rt, pg_type result + WHERE o.oprkind='l' AND o.oprright = rt.oid + AND o.oprresult = result.oid + ORDER BY operand""") + return result + +# list all binary operators +def list_binary_operator(pgcnx): + result = pgcnx.query("""SELECT o.oprname AS binary_op, + rt.typname AS right_opr, lt.typname AS left_opr, + result.typname AS return_type + FROM pg_operator o, pg_type rt, pg_type lt, pg_type result + WHERE o.oprkind = 'b' AND o.oprright = rt.oid + AND o.oprleft = lt.oid AND o.oprresult = result.oid""") + return result + +# returns the name, args and return type from all function of lang l +def list_lang_func(pgcnx, l): + result = pgcnx.query("""SELECT p.proname, p.pronargs, t.typname + FROM pg_proc p, pg_language l, pg_type t + WHERE p.prolang = l.oid AND p.prorettype = t.oid + AND l.lanname = '%s' + ORDER BY proname""" % l) + return result + +# lists all the aggregate functions and the type to which they can be applied +def list_agg_func(pgcnx): + result = pgcnx.query("""SELECT p.proname, t.typname + FROM pg_aggregate a, pg_proc p, pg_type t + WHERE a.aggfnoid = p.oid + and p.proargtypes[0] = t.oid + ORDER BY proname, typname""") + return result + +# lists all the operator classes that can be used with each access method as +# well as the operators that can be used with the respective operator classes +def list_op_class(pgcnx): + result = pgcnx.query("""SELECT am.amname, opc.opcname, opr.oprname + FROM pg_am am, pg_amop amop, pg_opclass opc, pg_operator opr + WHERE amop.amopid = am.oid and amop.amopclaid = opc.oid + AND amop.amopopr = opr.oid order by amname, opcname, oprname""") + return result + +# demo function - runs all examples +def demo(pgcnx): + import sys, os + save_stdout = sys.stdout + sys.stdout = os.popen("more", "w") + print "Listing simple indices ..." + print list_simple_ind(pgcnx) + print "Listing all attributes ..." + print list_all_attr(pgcnx) + print "Listing all user-defined base types ..." + print list_user_base_type(pgcnx) + print "Listing all left-unary operators defined ..." + print list_left_unary_operator(pgcnx) + print "Listing all right-unary operators defined ..." + print list_right_unary_operator(pgcnx) + print "Listing all binary operators ..." + print list_binary_operator(pgcnx) + print "Listing C external function linked ..." + print list_lang_func(pgcnx, 'C') + print "Listing C internal functions ..." + print list_lang_func(pgcnx, 'internal') + print "Listing SQL functions defined ..." + print list_lang_func(pgcnx, 'sql') + print "Listing 'aggregate functions' ..." + print list_agg_func(pgcnx) + print "Listing 'operator classes' ..." + print list_op_class(pgcnx) + del sys.stdout + sys.stdout = save_stdout diff --git a/db/tests/helge/chkfolderinfoperf.py b/db/tests/helge/chkfolderinfoperf.py new file mode 100755 index 00000000..e4700e28 --- /dev/null +++ b/db/tests/helge/chkfolderinfoperf.py @@ -0,0 +1,23 @@ +#!/usr/bin/python + +import pg, time + +USER="agenor" +HOST="localhost" +DB="agenor_fldinfodb" +TABLE="SOGo_folder_info" + +db = pg.DB(DB, HOST, 5432, "", "", USER); +print "connection:", db + +def timeSelect(SELECT, WHERE=None): + SQL="SELECT %s FROM %s" % ( SELECT, TABLE ) + if not WHERE is None: SQL="%s WHERE %s" % ( SQL, WHERE ) + starttime=time.time() + res = db.query(SQL) + endtime=time.time() + print "perf '%s', %s: %.2fms" % ( SELECT, WHERE, (endtime-starttime)*1000) + +timeSelect("COUNT(*)") +timeSelect("c_tablename, c_dbname") +timeSelect("c_tablename, c_dbname", "c_foldername='privcal_99827'") diff --git a/db/tests/helge/chkhugeperf.py b/db/tests/helge/chkhugeperf.py new file mode 100755 index 00000000..36c588cc --- /dev/null +++ b/db/tests/helge/chkhugeperf.py @@ -0,0 +1,24 @@ +#!/usr/bin/python + +import pg, time + +USER="agenor" +HOST="localhost" +DB="agenor_testhugeperf" +TABLE="SOGo_huge_quick" + +db = pg.DB(DB, HOST, 5432, "", "", USER); +print "connection:", db + +def timeSelect(SELECT, WHERE=None): + SQL="SELECT %s FROM %s" % ( SELECT, TABLE ) + if not WHERE is None: SQL="%s WHERE %s" % ( SQL, WHERE ) + starttime=time.time() + res = db.query(SQL) + endtime=time.time() + print "perf '%s', %s: %.2fms" % ( SELECT, WHERE, (endtime-starttime)*1000) + +timeSelect("COUNT(*)") +timeSelect("c_pkey, c_sourceid, c_startdate, c_enddate", + "c_startdate > 1085068363 AND c_startdate < 1085068373") +timeSelect("c_sourceid") diff --git a/db/tests/helge/fill_hugeperf.py b/db/tests/helge/fill_hugeperf.py new file mode 100755 index 00000000..26a379ee --- /dev/null +++ b/db/tests/helge/fill_hugeperf.py @@ -0,0 +1,80 @@ +#!/usr/bin/python + +import pg, time, sys + +USER="agenor" +HOST="localhost" +DB="agenor_testhugeperf" +QUICKTABLE="SOGo_huge_quick" +BLOBTABLE="SOGo_huge_ical" + +db = pg.DB(DB, HOST, 5432, "", "", USER); +print "connection:", db + +# index, index, start, end, index, +QUICKTEMPLATE="""INSERT INTO %s ( + c_pkey, c_sourceid, c_startdate, c_enddate, c_title, c_attendees, + c_isallday, c_sequenceid, c_generation +) VALUES ( + %i, 'longsource%iid', %i, %i, 'long title %i', + 'hh@skyrix.com,mm@skyrix,je@skyrix.com,fr@skyrix.com,jm@skyrix.com,hh@skyrix.com,mm@skyrix,je@skyrix.com,fr@skyrix.com,jm@skyrix.com,hh@skyrix.com,mm@skyrix,je@skyrix.com,fr@skyrix.com,jm@skyrix.com', + 0, 0, 1 +);""" + +# index, ical +ICALTEMPLATE="INSERT INTO %s ( c_pkey, c_data ) VALUES ( %i, '%s' );" + +ICALFILE="""BEGIN:VEVENT +DURATION:PT1H +ATTENDEE;CN="Elke Bethke";DIR="addressbook://B156F3F0-9CFD-11D8-8561-000 + D93C1A604:ABPerson":mailto:E.Bethke@Sachsen-Anhalt-Lotto.de +ATTENDEE;CN="Erik Doernenburg";DIR="addressbook://B15FCB0F-9CFD-11D8-8561 + -000D93C1A604:ABPerson":mailto:erik@x101.net +ATTENDEE;CN="Christian Schnelle";DIR="addressbook://B1418D4E-9CFD-11D8-8 + 561-000D93C1A604:ABPerson":mailto:cs@enervation.de +ATTENDEE;CN="Chris Herrenberger";DIR="addressbook://B14A390C-9CFD-11D8-8 + 561-000D93C1A604:ABPerson":invalid:nomail +ATTENDEE;CN="Horst Parplies";DIR="addressbook://B19B47E5-9CFD-11D8-8561- + 000D93C1A604:ABPerson":mailto:horst.parplies@freenet.de +ATTENDEE;CN="Imdat Solak";DIR="addressbook://B19EDB62-9CFD-11D8-8561-000 + D93C1A604:ABPerson":mailto:imdat@solak.de +ATTENDEE;CN="Jens Enders";DIR="addressbook://B1B6819F-9CFD-11D8-8561-000 + D93C1A604:ABPerson":mailto:jens.enders@skyrix.com +ATTENDEE;CN="Jens Muenster";DIR="addressbook://B1BBA42E-9CFD-11D8-8561-00 + 0D93C1A604:ABPerson":mailto:jens.muenster@skyrix.com +ATTENDEE;CN="Laurent Pierre";DIR="addressbook://9337C270-A825-11D8-B930- + 000D93C1A604:ABPerson":mailto:laurent.pierre@linagora.com +ATTENDEE;CN="Marcel Weiher";DIR="addressbook://B1F9BB12-9CFD-11D8-8561-0 + 00D93C1A604:ABPerson":mailto:marcel@metaobject.co +DTSTAMP:20040520T140002Z +UID:BD91C454-AA65-11D8-84CA-000D93C1A604 +SEQUENCE:3 +STATUS:CONFIRMED +DTSTART;TZID=Europe/Berlin:20040618T160000 +SUMMARY:SIZE EVENT +X-WR-ITIPSTATUSML:UNCLEAN +END:VEVENT +""" + + +# ******************** INSERT ******************** + +FROM=int(sys.argv[1]) +TO=FROM+1000000 +#FROM=1 +#TO=1000000 + +timingstart=time.time() + +for i in range(FROM, TO): + start=time.time() + end=start+(60 * 30) + + QSQL = QUICKTEMPLATE % ( QUICKTABLE, i, i, start, end, i, ) + BSQL = ICALTEMPLATE % ( BLOBTABLE, i, ICALFILE ) + + if i % 10000 == 0: + print "%i (%.2fs): quick %s" % ( i, time.time()-timingstart, QSQL ) + #print "%i: blob %s" % ( i, BSQL ) + db.query(QSQL + BSQL) + diff --git a/db/tests/helge/fillfolderinfo.py b/db/tests/helge/fillfolderinfo.py new file mode 100755 index 00000000..2fc12c42 --- /dev/null +++ b/db/tests/helge/fillfolderinfo.py @@ -0,0 +1,19 @@ +#!/usr/bin/python + +import pg + +USER="agenor" +HOST="localhost" +DB="agenor_fldinfodb" +TABLE="SOGo_folder_info" + +db = pg.DB(DB, HOST, 5432, "", "", USER); +print "connection:", db + +for i in range(110001, 250000): + SQL=("INSERT INTO %s ( c_foldername, c_tablename, c_dbname ) " + \ + "VALUES ( 'privcal_%i', 'agenor_tab_%i', 'agenortabledb' );") % \ + ( TABLE, i, i, ) + if i % 1000 == 0: + print "%i: %s" % ( i, SQL ) + db.query(SQL) diff --git a/db/tests/helge/makedb.py b/db/tests/helge/makedb.py new file mode 100755 index 00000000..51139508 --- /dev/null +++ b/db/tests/helge/makedb.py @@ -0,0 +1,12 @@ +#!/usr/bin/python + +import os, sys + +HOST="localhost" +USER="agenor" +DB="blah2" + +for i in range(920, 10000): + DB="agenor%i" % ( i, ) + res=os.system("createdb -h %s -U %s %s" % ( HOST, USER, DB )) + print "%s res: %i" % ( DB, res ) diff --git a/db/tests/helge/maketables.py b/db/tests/helge/maketables.py new file mode 100755 index 00000000..16c08996 --- /dev/null +++ b/db/tests/helge/maketables.py @@ -0,0 +1,23 @@ +#!/usr/bin/python + +import os, sys + +HOST="localhost" +USER="agenor" +DB="agenortabledb" + +for i in range(10000, 60000): + NEWTABLE="agenor_tab_%i" % ( i, ) + + TABLECREATE="""CREATE TABLE %s + ( pkey INT PRIMARY KEY, + startdate INT NOT NULL, + endate INT NOT NULL, + title VARCHAR(1000) NOT NULL, + participants VARCHAR(100000) NOT NULL);""" % ( NEWTABLE, ) + + CALL="echo '%s' | psql -h %s %s %s" % ( TABLECREATE, HOST, DB, USER ) + res=os.system(CALL) + #res = 1 + #print "CALL:", CALL + print "%s res: %i" % ( NEWTABLE, res ) diff --git a/db/tests/helge/makeusers.py b/db/tests/helge/makeusers.py new file mode 100755 index 00000000..a8ac623e --- /dev/null +++ b/db/tests/helge/makeusers.py @@ -0,0 +1,12 @@ +#!/usr/bin/python + +import os, sys + +HOST="localhost" +USER="agenor" +DB="" + +for i in range(1, 10000): + NEWUSER="agenor%i" % ( i, ) + res=os.system("createuser -A -D -h %s -U %s %s" % ( HOST, USER, NEWUSER )) + print "%s res: %i" % ( NEWUSER, res )