--- /dev/null
+# $Id$
+
+Issues
+- folder generation?
+ - addition der item-generations geht nicht, weil auch records geloescht
+ werden koennen
+- separate tabelle fuer "proposed" appointments?
+ - damit wir die Apts nicht schicken muessen und die erst im scheduler
+ angezeigt werden, wenn sie aktuell sind
+
+Move Databases
+==============
+
+HOST="localhost"
+USER="agenor"
+
+DB="blah2"
+DB="agenor%i" % ( i, )
+
+DB="agenortabledb"
+NEWTABLE="agenor_tab_%i" % ( i, )
+
+DB="agenor_fldinfodb"
+
+DB="agenor_testhugeperf"
+
+
+Schemas
+=======
+
+CREATE TABLE SOGo_folder_info (
+ c_foldername VARCHAR(255) NOT NULL,
+ c_tablename VARCHAR(255) NOT NULL,
+ c_dbname VARCHAR(255) NOT NULL
+);
+INSERT INTO SOGo_folder_info (c_foldername,c_tablename,c_dbname)
+ VALUES ('hh calendar', 'agenor_tab_105', 'agenortabledb');
+
+CREATE UNIQUE INDEX SOGo_folder_info_idx
+ ON SOGo_folder_info
+ USING BTREE(c_foldername);
+(ca 5s)
+
+---
+ CREATE TABLE SOGo_huge_ical (
+ c_pkey INT PRIMARY KEY,
+ c_data VARCHAR(32000) NOT NULL
+ );
+ CREATE TABLE SOGo_huge_quick (
+ c_pkey INT PRIMARY KEY,
+ c_sourceid VARCHAR(255) NOT NULL,
+ c_startdate INT NOT NULL,
+ c_enddate INT NOT NULL,
+ c_title VARCHAR(255) NOT NULL,
+ c_attendees VARCHAR(4096) NOT NULL,
+ c_isallday INT NOT NULL,
+ c_sequenceid INT NOT NULL,
+ c_generation INT NOT NULL
+ );
+CREATE INDEX SOGo_huge_quick_idx
+ ON SOGo_huge_quick
+ USING BTREE(c_startdate);
+(ca 15s on 1.000.000)
+---
+
+
+Performance
+===========
+agenor_fldinfodb:
+ 10000 Folder Info Entries, kein Index: 71s
+ 100000 Folder Info Entries: 12:09m, 729s, 137 inserts/s
+ 5992424=>6001088, diff 8664KB data size (von ca 16000 auf 110000)
+ ~94 byte per row (raw: ~12+14+13=39 byte)
+ 110001-250000 Folder Info Entries: 15:59m, 959s, 145 inserts/s
+ 6001088-6014316, diff 13228KB data size (~96 byte per row)
+
+ ohne Index, via Python:
+ COUNT(*) => 360ms
+ c_tablename, c_dbname | * => 1128ms
+ c_tablename, c_dbname | c_foldername='privcal_99827' => 345ms
+ mit Index, via Python
+ COUNT(*) => 350ms
+ c_tablename, c_dbname | * => 1124ms
+ c_tablename, c_dbname | c_foldername='privcal_99827' => 18,5,5ms
+
+agenor_testhugeperf:
+ 1.000.000 entries, kein Index:
+ 10000=79.37s, 20000=162s, 30000=245s,340000=2831s,790000=6670s
+ (~120 rows per sec) = > ca 2h fuer 1.000.000, 20h fuer 10.000.000
+ 30000=65MB => ~2KB per Record => ~2GB fuer 1.000.000
+ 220000=440MB,810000=1.55GB,1.000.000=
+ ~1.92GB x 20 = 40GB
+ duration:~2:50h
+ ohne Index, via Python:
+ COUNT(*) => 20.8s
+ pkey,sourceid,start,end | c_start>x&c_end<x+10s => 13.4s
+ c_sourceid | * => 10.5s
+ mit Index, via Python
+ COUNT(*) => 9.7s,3s,2.5s
+ pkey,sourceid,start,end | c_start>x&c_end<x+10s => 86ms,59ms,17ms,17ms
+ c_sourceid | * => 9.3s,4.8s,4.8s
+ vacuum analyze: 30s?
+ ohne fsync, 2 gleichzeitig
+ - 1108 rows pro sec in einer connection! (1.000.000=ca 15min)
+ 53:59m
+ 53:52m
+ ~8 gleichzeitig
+ ~20:00:-
+
+http://www.varlena.com/varlena/GeneralBits/Tidbits/perf.html
+http://www.varlena.com/varlena/GeneralBits/Tidbits/annotated_conf_e.html
+http://www.lyris.com/lm_help/7.8/tuning_postgresql.html
+http://www.linuxjournal.com/article.php?sid=4791
+http://www.argudo.org/postgresql/soft-tuning.php#s2
+
+PostgreSQL Index
+================
+http://www.postgresql.org/docs/current/static/sql-createindex.html
+http://postgis.refractions.net/docs/x511.html
+
+"After building an index, it is important to force PostgreSQL to collect table statistics, which are used to optimize query plans: VACUUM ANALIZE;"
+
+PostgreSQL provides the index methods B-tree, R-tree, hash, and GiST
+- only the B-tree and GiST index methods support multicolumn indexes
+
+---snip---
+An index field can be an expression computed from the values of one or more
+columns of the table row. This feature can be used to obtain fast access to
+data based on some transformation of the basic data. For example, an index
+computed on upper(col) would allow the clause WHERE upper(col) = 'JIM' to use
+an index.
+---snap---
+
+---snip---
+Note: Because of the limited utility of hash indexes, a B-tree index should
+generally be preferred over a hash index. We do not have sufficient evidence
+that hash indexes are actually faster than B-trees even for = comparisons.
+Moreover, hash indexes require coarser locks; see Section 9.7.
+---snap--
+
+PostgreSQL Arrays
+=================
+---snip---
+It all depends, I have found array operations to be slow. So if you have just
+a few elements, like less than 10 then arrays are ok, but with more e.g. 30+
+elements, as in my case, imho the whole exercise is not really feasable. I am
+going to re-design the schema to get rid of the arrays as soon as I have a
+moment. Also the code around that part of PostgreSQL has not been visited for
+a fair while and needs some polishing up. I'd avoid them.
+---snap---
--- /dev/null
+#! /usr/bin/env python
+# advanced.py - demo of advanced features of PostGres. Some may not be ANSI.
+# inspired from the Postgres tutorial
+# adapted to Python 1995 by Pascal Andre
+
+print """
+__________________________________________________________________
+MODULE ADVANCED.PY : ADVANCED POSTGRES SQL COMMANDS TUTORIAL
+
+This module is designed for being imported from python prompt
+
+In order to run the samples included here, first create a connection
+using : cnx = advanced.DB(...)
+
+The "..." should be replaced with whatever arguments you need to open an
+existing database. Usually all you need is the name of the database and,
+in fact, if it is the same as your login name, you can leave it empty.
+
+then start the demo with: advanced.demo(cnx)
+__________________________________________________________________
+"""
+
+from pg import DB
+import sys
+
+# waits for a key
+def wait_key():
+ print "Press <enter>"
+ sys.stdin.read(1)
+
+# inheritance features
+def inherit_demo(pgcnx):
+ print "-----------------------------"
+ print "-- Inheritance:"
+ print "-- a table can inherit from zero or more tables. A query"
+ print "-- can reference either all rows of a table or all rows "
+ print "-- of a table plus all of its descendants."
+ print "-----------------------------"
+ print
+ print "-- For example, the capitals table inherits from cities table."
+ print "-- (It inherits all data fields from cities.)"
+ print
+ print "CREATE TABLE cities ("
+ print " name text,"
+ print " population float8,"
+ print " altitude int"
+ print ")"
+ print
+ print "CREATE TABLE capitals ("
+ print " state varchar(2)"
+ print ") INHERITS (cities)"
+ pgcnx.query("""CREATE TABLE cities (
+ name text,
+ population float8,
+ altitude int)""")
+ pgcnx.query("""CREATE TABLE capitals (
+ state varchar(2)) INHERITS (cities)""")
+ wait_key()
+ print
+ print "-- now, let's populate the tables"
+ print
+ print "INSERT INTO cities VALUES ('San Francisco', 7.24E+5, 63)"
+ print "INSERT INTO cities VALUES ('Las Vegas', 2.583E+5, 2174)"
+ print "INSERT INTO cities VALUES ('Mariposa', 1200, 1953)"
+ print
+ print "INSERT INTO capitals VALUES ('Sacramento', 3.694E+5, 30, 'CA')"
+ print "INSERT INTO capitals VALUES ('Madison', 1.913E+5, 845, 'WI')"
+ print
+ pgcnx.query("INSERT INTO cities VALUES ('San Francisco', 7.24E+5, 63)")
+ pgcnx.query("INSERT INTO cities VALUES ('Las Vegas', 2.583E+5, 2174)")
+ pgcnx.query("INSERT INTO cities VALUES ('Mariposa', 1200, 1953)")
+ pgcnx.query("INSERT INTO capitals VALUES ('Sacramento',3.694E+5,30,'CA')")
+ pgcnx.query("INSERT INTO capitals VALUES ('Madison', 1.913E+5, 845, 'WI')")
+ print
+ print "SELECT * FROM cities"
+ print pgcnx.query("SELECT * FROM cities")
+ print "SELECT * FROM capitals"
+ print pgcnx.query("SELECT * FROM capitals")
+ print
+ print "-- like before, a regular query references rows of the base"
+ print "-- table only"
+ print
+ print "SELECT name, altitude"
+ print "FROM cities"
+ print "WHERE altitude > 500;"
+ print pgcnx.query("""SELECT name, altitude
+ FROM cities
+ WHERE altitude > 500""")
+ print
+ print "-- on the other hand, you can find all cities, including "
+ print "-- capitals, that are located at an altitude of 500 'ft "
+ print "-- or higher by:"
+ print
+ print "SELECT c.name, c.altitude"
+ print "FROM cities* c"
+ print "WHERE c.altitude > 500"
+ print pgcnx.query("""SELECT c.name, c.altitude
+ FROM cities* c
+ WHERE c.altitude > 500""")
+
+# arrays attributes
+def array_demo(pgcnx):
+ print "----------------------"
+ print "-- Arrays:"
+ print "-- attributes can be arrays of base types or user-defined "
+ print "-- types"
+ print "----------------------"
+ print
+ print "CREATE TABLE sal_emp ("
+ print " name text,"
+ print " pay_by_quarter int4[],"
+ print " pay_by_extra_quarter int8[],"
+ print " schedule text[][]"
+ print ")"
+ pgcnx.query("""CREATE TABLE sal_emp (
+ name text,
+ pay_by_quarter int4[],
+ pay_by_extra_quarter int8[],
+ schedule text[][])""")
+ wait_key()
+ print
+ print "-- insert instances with array attributes. "
+ print " Note the use of braces"
+ print
+ print "INSERT INTO sal_emp VALUES ("
+ print " 'Bill',"
+ print " '{10000,10000,10000,10000}',"
+ print " '{9223372036854775800,9223372036854775800,9223372036854775800}',"
+ print " '{{\"meeting\", \"lunch\"}, {}}')"
+ print
+ print "INSERT INTO sal_emp VALUES ("
+ print " 'Carol',"
+ print " '{20000,25000,25000,25000}',"
+ print " '{9223372036854775807,9223372036854775807,9223372036854775807}',"
+ print " '{{\"talk\", \"consult\"}, {\"meeting\"}}')"
+ print
+ pgcnx.query("""INSERT INTO sal_emp VALUES (
+ 'Bill', '{10000,10000,10000,10000}',
+ '{9223372036854775800,9223372036854775800,9223372036854775800}',
+ '{{\"meeting\", \"lunch\"}, {}}')""")
+ pgcnx.query("""INSERT INTO sal_emp VALUES (
+ 'Carol', '{20000,25000,25000,25000}',
+ '{9223372036854775807,9223372036854775807,9223372036854775807}',
+ '{{\"talk\", \"consult\"}, {\"meeting\"}}')""")
+ wait_key()
+ print
+ print "----------------------"
+ print "-- queries on array attributes"
+ print "----------------------"
+ print
+ print "SELECT name FROM sal_emp WHERE"
+ print " sal_emp.pay_by_quarter[1] <> sal_emp.pay_by_quarter[2]"
+ print
+ print pgcnx.query("""SELECT name FROM sal_emp WHERE
+ sal_emp.pay_by_quarter[1] <> sal_emp.pay_by_quarter[2]""")
+ print
+ print pgcnx.query("""SELECT name FROM sal_emp WHERE
+ sal_emp.pay_by_extra_quarter[1] <> sal_emp.pay_by_extra_quarter[2]""")
+ print
+ print "-- retrieve third quarter pay of all employees"
+ print
+ print "SELECT sal_emp.pay_by_quarter[3] FROM sal_emp"
+ print
+ print pgcnx.query("SELECT sal_emp.pay_by_quarter[3] FROM sal_emp")
+ print
+ print "-- retrieve third quarter extra pay of all employees"
+ print
+ print "SELECT sal_emp.pay_by_extra_quarter[3] FROM sal_emp"
+ print pgcnx.query("SELECT sal_emp.pay_by_extra_quarter[3] FROM sal_emp")
+ print
+ print "-- retrieve first two quarters of extra quarter pay of all employees"
+ print
+ print "SELECT sal_emp.pay_by_extra_quarter[1:2] FROM sal_emp"
+ print
+ print pgcnx.query("SELECT sal_emp.pay_by_extra_quarter[1:2] FROM sal_emp")
+ print
+ print "-- select subarrays"
+ print
+ print "SELECT sal_emp.schedule[1:2][1:1] FROM sal_emp WHERE"
+ print " sal_emp.name = 'Bill'"
+ print pgcnx.query("SELECT sal_emp.schedule[1:2][1:1] FROM sal_emp WHERE " \
+ "sal_emp.name = 'Bill'")
+
+# base cleanup
+def demo_cleanup(pgcnx):
+ print "-- clean up (you must remove the children first)"
+ print "DROP TABLE sal_emp"
+ print "DROP TABLE capitals"
+ print "DROP TABLE cities;"
+ pgcnx.query("DROP TABLE sal_emp")
+ pgcnx.query("DROP TABLE capitals")
+ pgcnx.query("DROP TABLE cities")
+
+# main demo function
+def demo(pgcnx):
+ inherit_demo(pgcnx)
+ array_demo(pgcnx)
+ demo_cleanup(pgcnx)
--- /dev/null
+#! /usr/bin/env python
+# basics.py - basic SQL commands tutorial
+# inspired from the Postgres95 tutorial
+# adapted to Python 1995 by Pascal ANDRE
+
+print """
+__________________________________________________________________
+MODULE BASICS.PY : BASIC POSTGRES SQL COMMANDS TUTORIAL
+
+This module is designed for being imported from python prompt
+
+In order to run the samples included here, first create a connection
+using : cnx = basics.DB(...)
+
+The "..." should be replaced with whatever arguments you need to open an
+existing database. Usually all you need is the name of the database and,
+in fact, if it is the same as your login name, you can leave it empty.
+
+then start the demo with: basics.demo(cnx)
+__________________________________________________________________
+"""
+
+from pg import DB
+import sys
+
+# waits for a key
+def wait_key():
+ print "Press <enter>"
+ sys.stdin.read(1)
+
+# table creation commands
+def create_table(pgcnx):
+ print "-----------------------------"
+ print "-- Creating a table:"
+ print "-- a CREATE TABLE is used to create base tables. POSTGRES"
+ print "-- SQL has its own set of built-in types. (Note that"
+ print "-- keywords are case-insensitive but identifiers are "
+ print "-- case-sensitive.)"
+ print "-----------------------------"
+ print
+ print "Sending query :"
+ print "CREATE TABLE weather ("
+ print " city varchar(80),"
+ print " temp_lo int,"
+ print " temp_hi int,"
+ print " prcp float8,"
+ print " date date"
+ print ")"
+ pgcnx.query("""CREATE TABLE weather (city varchar(80), temp_lo int,
+ temp_hi int, prcp float8, date date)""")
+ print
+ print "Sending query :"
+ print "CREATE TABLE cities ("
+ print " name varchar(80),"
+ print " location point"
+ print ")"
+ pgcnx.query("""CREATE TABLE cities (
+ name varchar(80),
+ location point)""")
+
+# data insertion commands
+def insert_data(pgcnx):
+ print "-----------------------------"
+ print "-- Inserting data:"
+ print "-- an INSERT statement is used to insert a new row into"
+ print "-- a table. There are several ways you can specify what"
+ print "-- columns the data should go to."
+ print "-----------------------------"
+ print
+ print "-- 1. the simplest case is when the list of value correspond to"
+ print "-- the order of the columns specified in CREATE TABLE."
+ print
+ print "Sending query :"
+ print "INSERT INTO weather "
+ print " VALUES ('San Francisco', 46, 50, 0.25, '11/27/1994')"
+ pgcnx.query("""INSERT INTO weather
+ VALUES ('San Francisco', 46, 50, 0.25, '11/27/1994')""")
+ print
+ print "Sending query :"
+ print "INSERT INTO cities "
+ print " VALUES ('San Francisco', '(-194.0, 53.0)')"
+ pgcnx.query("""INSERT INTO cities
+ VALUES ('San Francisco', '(-194.0, 53.0)')""")
+ print
+ wait_key()
+ print "-- 2. you can also specify what column the values correspond "
+ print " to. (The columns can be specified in any order. You may "
+ print " also omit any number of columns. eg. unknown precipitation"
+ print " below)"
+ print "Sending query :"
+ print "INSERT INTO weather (city, temp_lo, temp_hi, prcp, date)"
+ print " VALUES ('San Francisco', 43, 57, 0.0, '11/29/1994')"
+ pgcnx.query("INSERT INTO weather (date, city, temp_hi, temp_lo)" \
+ "VALUES ('11/29/1994', 'Hayward', 54, 37)")
+
+# direct selection commands
+def select_data1(pgcnx):
+ print "-----------------------------"
+ print "-- Retrieving data:"
+ print "-- a SELECT statement is used for retrieving data. The "
+ print "-- basic syntax is:"
+ print "-- SELECT columns FROM tables WHERE predicates"
+ print "-----------------------------"
+ print
+ print "-- a simple one would be the query:"
+ print "SELECT * FROM weather"
+ print
+ print "The result is :"
+ q = pgcnx.query("SELECT * FROM weather")
+ print q
+ print
+ print "-- you may also specify expressions in the target list (the "
+ print "-- 'AS column' specifies the column name of the result. It is "
+ print "-- optional.)"
+ print "The query :"
+ print " SELECT city, (temp_hi+temp_lo)/2 AS temp_avg, date "
+ print " FROM weather"
+ print "Gives :"
+ print pgcnx.query("""SELECT city, (temp_hi+temp_lo)/2
+ AS temp_avg, date FROM weather""")
+ print
+ print "-- if you want to retrieve rows that satisfy certain condition"
+ print "-- (ie. a restriction), specify the condition in WHERE. The "
+ print "-- following retrieves the weather of San Francisco on rainy "
+ print "-- days."
+ print "SELECT *"
+ print "FROM weather"
+ print "WHERE city = 'San Francisco' "
+ print " and prcp > 0.0"
+ print pgcnx.query("""SELECT * FROM weather WHERE city = 'San Francisco'
+ AND prcp > 0.0""")
+ print
+ print "-- here is a more complicated one. Duplicates are removed when "
+ print "-- DISTINCT is specified. ORDER BY specifies the column to sort"
+ print "-- on. (Just to make sure the following won't confuse you, "
+ print "-- DISTINCT and ORDER BY can be used separately.)"
+ print "SELECT DISTINCT city"
+ print "FROM weather"
+ print "ORDER BY city;"
+ print pgcnx.query("SELECT DISTINCT city FROM weather ORDER BY city")
+
+# selection to a temporary table
+def select_data2(pgcnx):
+ print "-----------------------------"
+ print "-- Retrieving data into other classes:"
+ print "-- a SELECT ... INTO statement can be used to retrieve "
+ print "-- data into another class."
+ print "-----------------------------"
+ print
+ print "The query :"
+ print "SELECT * INTO TABLE temptab "
+ print "FROM weather"
+ print "WHERE city = 'San Francisco' "
+ print " and prcp > 0.0"
+ pgcnx.query("""SELECT * INTO TABLE temptab FROM weather
+ WHERE city = 'San Francisco' and prcp > 0.0""")
+ print "Fills the table temptab, that can be listed with :"
+ print "SELECT * from temptab"
+ print pgcnx.query("SELECT * from temptab")
+
+# aggregate creation commands
+def create_aggregate(pgcnx):
+ print "-----------------------------"
+ print "-- Aggregates"
+ print "-----------------------------"
+ print
+ print "Let's consider the query :"
+ print "SELECT max(temp_lo)"
+ print "FROM weather;"
+ print pgcnx.query("SELECT max(temp_lo) FROM weather")
+ print
+ print "-- Aggregate with GROUP BY"
+ print "SELECT city, max(temp_lo)"
+ print "FROM weather "
+ print "GROUP BY city;"
+ print pgcnx.query( """SELECT city, max(temp_lo)
+ FROM weather GROUP BY city""")
+
+# table join commands
+def join_table(pgcnx):
+ print "-----------------------------"
+ print "-- Joining tables:"
+ print "-- queries can access multiple tables at once or access"
+ print "-- the same table in such a way that multiple instances"
+ print "-- of the table are being processed at the same time."
+ print "-----------------------------"
+ print
+ print "-- suppose we want to find all the records that are in the "
+ print "-- temperature range of other records. W1 and W2 are aliases "
+ print "--for weather."
+ print
+ print "SELECT W1.city, W1.temp_lo, W1.temp_hi, "
+ print " W2.city, W2.temp_lo, W2.temp_hi"
+ print "FROM weather W1, weather W2"
+ print "WHERE W1.temp_lo < W2.temp_lo "
+ print " and W1.temp_hi > W2.temp_hi"
+ print
+ print pgcnx.query("""SELECT W1.city, W1.temp_lo, W1.temp_hi,
+ W2.city, W2.temp_lo, W2.temp_hi FROM weather W1, weather W2
+ WHERE W1.temp_lo < W2.temp_lo and W1.temp_hi > W2.temp_hi""")
+ print
+ print "-- let's join two tables. The following joins the weather table"
+ print "-- and the cities table."
+ print
+ print "SELECT city, location, prcp, date"
+ print "FROM weather, cities"
+ print "WHERE name = city"
+ print
+ print pgcnx.query("""SELECT city, location, prcp, date FROM weather, cities
+ WHERE name = city""")
+ print
+ print "-- since the column names are all different, we don't have to "
+ print "-- specify the table name. If you want to be clear, you can do "
+ print "-- the following. They give identical results, of course."
+ print
+ print "SELECT w.city, c.location, w.prcp, w.date"
+ print "FROM weather w, cities c"
+ print "WHERE c.name = w.city;"
+ print
+ print pgcnx.query("""SELECT w.city, c.location, w.prcp, w.date
+ FROM weather w, cities c WHERE c.name = w.city""")
+
+# data updating commands
+def update_data(pgcnx):
+ print "-----------------------------"
+ print "-- Updating data:"
+ print "-- an UPDATE statement is used for updating data. "
+ print "-----------------------------"
+ print
+ print "-- suppose you discover the temperature readings are all off by"
+ print "-- 2 degrees as of Nov 28, you may update the data as follow:"
+ print
+ print "UPDATE weather"
+ print " SET temp_hi = temp_hi - 2, temp_lo = temp_lo - 2"
+ print " WHERE date > '11/28/1994'"
+ print
+ pgcnx.query("""UPDATE weather
+ SET temp_hi = temp_hi - 2, temp_lo = temp_lo - 2
+ WHERE date > '11/28/1994'""")
+ print
+ print "SELECT * from weather"
+ print pgcnx.query("SELECT * from weather")
+
+# data deletion commands
+def delete_data(pgcnx):
+ print "-----------------------------"
+ print "-- Deleting data:"
+ print "-- a DELETE statement is used for deleting rows from a "
+ print "-- table."
+ print "-----------------------------"
+ print
+ print "-- suppose you are no longer interested in the weather of "
+ print "-- Hayward, you can do the following to delete those rows from"
+ print "-- the table"
+ print
+ print "DELETE FROM weather WHERE city = 'Hayward'"
+ pgcnx.query("DELETE FROM weather WHERE city = 'Hayward'")
+ print
+ print "SELECT * from weather"
+ print
+ print pgcnx.query("SELECT * from weather")
+ print
+ print "-- you can also delete all the rows in a table by doing the "
+ print "-- following. (This is different from DROP TABLE which removes "
+ print "-- the table in addition to the removing the rows.)"
+ print
+ print "DELETE FROM weather"
+ pgcnx.query("DELETE FROM weather")
+ print
+ print "SELECT * from weather"
+ print pgcnx.query("SELECT * from weather")
+
+# table removal commands
+def remove_table(pgcnx):
+ print "-----------------------------"
+ print "-- Removing the tables:"
+ print "-- DROP TABLE is used to remove tables. After you have"
+ print "-- done this, you can no longer use those tables."
+ print "-----------------------------"
+ print
+ print "DROP TABLE weather, cities, temptab"
+ pgcnx.query("DROP TABLE weather, cities, temptab")
+
+# main demo function
+def demo(pgcnx):
+ create_table(pgcnx)
+ wait_key()
+ insert_data(pgcnx)
+ wait_key()
+ select_data1(pgcnx)
+ select_data2(pgcnx)
+ create_aggregate(pgcnx)
+ join_table(pgcnx)
+ update_data(pgcnx)
+ delete_data(pgcnx)
+ remove_table(pgcnx)
--- /dev/null
+# func.py - demonstrate the use of SQL functions
+# inspired from the PostgreSQL tutorial
+# adapted to Python 1995 by Pascal ANDRE
+
+print """
+__________________________________________________________________
+MODULE FUNC.PY : SQL FUNCTION DEFINITION TUTORIAL
+
+This module is designed for being imported from python prompt
+
+In order to run the samples included here, first create a connection
+using : cnx = func.DB(...)
+
+The "..." should be replaced with whatever arguments you need to open an
+existing database. Usually all you need is the name of the database and,
+in fact, if it is the same as your login name, you can leave it empty.
+
+then start the demo with: func.demo(cnx)
+__________________________________________________________________
+"""
+
+from pg import DB
+import sys
+
+# waits for a key
+def wait_key():
+ print "Press <enter>"
+ sys.stdin.read(1)
+
+# basic functions declaration
+def base_func(pgcnx):
+ print "-----------------------------"
+ print "-- Creating SQL Functions on Base Types"
+ print "-- a CREATE FUNCTION statement lets you create a new "
+ print "-- function that can be used in expressions (in SELECT, "
+ print "-- INSERT, etc.). We will start with functions that "
+ print "-- return values of base types."
+ print "-----------------------------"
+ print
+ print "--"
+ print "-- let's create a simple SQL function that takes no arguments"
+ print "-- and returns 1"
+ print
+ print "CREATE FUNCTION one() RETURNS int4"
+ print " AS 'SELECT 1 as ONE' LANGUAGE 'sql'"
+ pgcnx.query("""CREATE FUNCTION one() RETURNS int4
+ AS 'SELECT 1 as ONE' LANGUAGE 'sql'""")
+ wait_key()
+ print
+ print "--"
+ print "-- functions can be used in any expressions (eg. in the target"
+ print "-- list or qualifications)"
+ print
+ print "SELECT one() AS answer"
+ print pgcnx.query("SELECT one() AS answer")
+ print
+ print "--"
+ print "-- here's how you create a function that takes arguments. The"
+ print "-- following function returns the sum of its two arguments:"
+ print
+ print "CREATE FUNCTION add_em(int4, int4) RETURNS int4"
+ print " AS 'SELECT $1 + $2' LANGUAGE 'sql'"
+ pgcnx.query("""CREATE FUNCTION add_em(int4, int4) RETURNS int4
+ AS 'SELECT $1 + $2' LANGUAGE 'sql'""")
+ print
+ print "SELECT add_em(1, 2) AS answer"
+ print pgcnx.query("SELECT add_em(1, 2) AS answer")
+
+# functions on composite types
+def comp_func(pgcnx):
+ print "-----------------------------"
+ print "-- Creating SQL Functions on Composite Types"
+ print "-- it is also possible to create functions that return"
+ print "-- values of composite types."
+ print "-----------------------------"
+ print
+ print "-- before we create more sophisticated functions, let's "
+ print "-- populate an EMP table"
+ print
+ print "CREATE TABLE EMP ("
+ print " name text,"
+ print " salary int4,"
+ print " age int4,"
+ print " dept varchar(16)"
+ print ")"
+ pgcnx.query("""CREATE TABLE EMP (
+ name text,
+ salary int4,
+ age int4,
+ dept varchar(16))""")
+ print
+ print "INSERT INTO EMP VALUES ('Sam', 1200, 16, 'toy')"
+ print "INSERT INTO EMP VALUES ('Claire', 5000, 32, 'shoe')"
+ print "INSERT INTO EMP VALUES ('Andy', -1000, 2, 'candy')"
+ print "INSERT INTO EMP VALUES ('Bill', 4200, 36, 'shoe')"
+ print "INSERT INTO EMP VALUES ('Ginger', 4800, 30, 'candy')"
+ pgcnx.query("INSERT INTO EMP VALUES ('Sam', 1200, 16, 'toy')")
+ pgcnx.query("INSERT INTO EMP VALUES ('Claire', 5000, 32, 'shoe')")
+ pgcnx.query("INSERT INTO EMP VALUES ('Andy', -1000, 2, 'candy')")
+ pgcnx.query("INSERT INTO EMP VALUES ('Bill', 4200, 36, 'shoe')")
+ pgcnx.query("INSERT INTO EMP VALUES ('Ginger', 4800, 30, 'candy')")
+ wait_key()
+ print
+ print "-- the argument of a function can also be a tuple. For "
+ print "-- instance, double_salary takes a tuple of the EMP table"
+ print
+ print "CREATE FUNCTION double_salary(EMP) RETURNS int4"
+ print " AS 'SELECT $1.salary * 2 AS salary' LANGUAGE 'sql'"
+ pgcnx.query("""CREATE FUNCTION double_salary(EMP) RETURNS int4
+ AS 'SELECT $1.salary * 2 AS salary' LANGUAGE 'sql'""")
+ print
+ print "SELECT name, double_salary(EMP) AS dream"
+ print "FROM EMP"
+ print "WHERE EMP.dept = 'toy'"
+ print pgcnx.query("""SELECT name, double_salary(EMP) AS dream
+ FROM EMP WHERE EMP.dept = 'toy'""")
+ print
+ print "-- the return value of a function can also be a tuple. However,"
+ print "-- make sure that the expressions in the target list is in the "
+ print "-- same order as the columns of EMP."
+ print
+ print "CREATE FUNCTION new_emp() RETURNS EMP"
+ print " AS 'SELECT \'None\'::text AS name,"
+ print " 1000 AS salary,"
+ print " 25 AS age,"
+ print " \'none\'::varchar(16) AS dept'"
+ print " LANGUAGE 'sql'"
+ pgcnx.query("""CREATE FUNCTION new_emp() RETURNS EMP
+ AS 'SELECT \\\'None\\\'::text AS name,
+ 1000 AS salary,
+ 25 AS age,
+ \\\'none\\\'::varchar(16) AS dept'
+ LANGUAGE 'sql'""")
+ wait_key()
+ print
+ print "-- you can then project a column out of resulting the tuple by"
+ print "-- using the \"function notation\" for projection columns. "
+ print "-- (ie. bar(foo) is equivalent to foo.bar) Note that we don't"
+ print "-- support new_emp().name at this moment."
+ print
+ print "SELECT name(new_emp()) AS nobody"
+ print pgcnx.query("SELECT name(new_emp()) AS nobody")
+ print
+ print "-- let's try one more function that returns tuples"
+ print "CREATE FUNCTION high_pay() RETURNS setof EMP"
+ print " AS 'SELECT * FROM EMP where salary > 1500'"
+ print " LANGUAGE 'sql'"
+ pgcnx.query("""CREATE FUNCTION high_pay() RETURNS setof EMP
+ AS 'SELECT * FROM EMP where salary > 1500'
+ LANGUAGE 'sql'""")
+ print
+ print "SELECT name(high_pay()) AS overpaid"
+ print pgcnx.query("SELECT name(high_pay()) AS overpaid")
+
+# function with multiple SQL commands
+def mult_func(pgcnx):
+ print "-----------------------------"
+ print "-- Creating SQL Functions with multiple SQL statements"
+ print "-- you can also create functions that do more than just a"
+ print "-- SELECT."
+ print "-----------------------------"
+ print
+ print "-- you may have noticed that Andy has a negative salary. We'll"
+ print "-- create a function that removes employees with negative "
+ print "-- salaries."
+ print
+ print "SELECT * FROM EMP"
+ print pgcnx.query("SELECT * FROM EMP")
+ print
+ print "CREATE FUNCTION clean_EMP () RETURNS int4"
+ print " AS 'DELETE FROM EMP WHERE EMP.salary <= 0"
+ print " SELECT 1 AS ignore_this'"
+ print " LANGUAGE 'sql'"
+ pgcnx.query("CREATE FUNCTION clean_EMP () RETURNS int4 AS 'DELETE FROM EMP WHERE EMP.salary <= 0; SELECT 1 AS ignore_this' LANGUAGE 'sql'")
+ print
+ print "SELECT clean_EMP()"
+ print pgcnx.query("SELECT clean_EMP()")
+ print
+ print "SELECT * FROM EMP"
+ print pgcnx.query("SELECT * FROM EMP")
+
+# base cleanup
+def demo_cleanup(pgcnx):
+ print "-- remove functions that were created in this file"
+ print
+ print "DROP FUNCTION clean_EMP()"
+ print "DROP FUNCTION high_pay()"
+ print "DROP FUNCTION new_emp()"
+ print "DROP FUNCTION add_em(int4, int4)"
+ print "DROP FUNCTION one()"
+ print
+ print "DROP TABLE EMP CASCADE"
+ pgcnx.query("DROP FUNCTION clean_EMP()")
+ pgcnx.query("DROP FUNCTION high_pay()")
+ pgcnx.query("DROP FUNCTION new_emp()")
+ pgcnx.query("DROP FUNCTION add_em(int4, int4)")
+ pgcnx.query("DROP FUNCTION one()")
+ pgcnx.query("DROP TABLE EMP CASCADE")
+
+# main demo function
+def demo(pgcnx):
+ base_func(pgcnx)
+ comp_func(pgcnx)
+ mult_func(pgcnx)
+ demo_cleanup(pgcnx)
--- /dev/null
+# syscat.py - parses some system catalogs
+# inspired from the PostgreSQL tutorial
+# adapted to Python 1995 by Pascal ANDRE
+
+print """
+__________________________________________________________________
+MODULE SYSCAT.PY : PARSES SOME POSTGRESQL SYSTEM CATALOGS
+
+This module is designed for being imported from python prompt
+
+In order to run the samples included here, first create a connection
+using : cnx = syscat.DB(...)
+
+The "..." should be replaced with whatever arguments you need to open an
+existing database. Usually all you need is the name of the database and,
+in fact, if it is the same as your login name, you can leave it empty.
+
+then start the demo with: syscat.demo(cnx)
+
+Some results may be empty, depending on your base status."
+
+__________________________________________________________________
+"""
+
+from pg import DB
+import sys
+
+# waits for a key
+def wait_key():
+ print "Press <enter>"
+ sys.stdin.read(1)
+
+# lists all simple indices
+def list_simple_ind(pgcnx):
+ result = pgcnx.query("""SELECT bc.relname AS class_name,
+ ic.relname AS index_name, a.attname
+ FROM pg_class bc, pg_class ic, pg_index i, pg_attribute a
+ WHERE i.indrelid = bc.oid AND i.indexrelid = bc.oid
+ AND i.indkey[0] = a.attnum AND a.attrelid = bc.oid
+ AND i.indproc = '0'::oid AND a.attisdropped = 'f'
+ ORDER BY class_name, index_name, attname""")
+ return result
+
+# list all user defined attributes and their type in user-defined classes
+def list_all_attr(pgcnx):
+ result = pgcnx.query("""SELECT c.relname, a.attname, t.typname
+ FROM pg_class c, pg_attribute a, pg_type t
+ WHERE c.relkind = 'r' and c.relname !~ '^pg_'
+ AND c.relname !~ '^Inv' and a.attnum > 0
+ AND a.attrelid = c.oid and a.atttypid = t.oid
+ AND a.attisdropped = 'f'
+ ORDER BY relname, attname""")
+ return result
+
+# list all user defined base type
+def list_user_base_type(pgcnx):
+ result = pgcnx.query("""SELECT u.usename, t.typname
+ FROM pg_type t, pg_user u
+ WHERE u.usesysid = int2in(int4out(t.typowner))
+ AND t.typrelid = '0'::oid and t.typelem = '0'::oid
+ AND u.usename <> 'postgres' order by usename, typname""")
+ return result
+
+# list all right-unary operators
+def list_right_unary_operator(pgcnx):
+ result = pgcnx.query("""SELECT o.oprname AS right_unary,
+ lt.typname AS operand, result.typname AS return_type
+ FROM pg_operator o, pg_type lt, pg_type result
+ WHERE o.oprkind='r' and o.oprleft = lt.oid
+ AND o.oprresult = result.oid
+ ORDER BY operand""")
+ return result
+
+# list all left-unary operators
+def list_left_unary_operator(pgcnx):
+ result = pgcnx.query("""SELECT o.oprname AS left_unary,
+ rt.typname AS operand, result.typname AS return_type
+ FROM pg_operator o, pg_type rt, pg_type result
+ WHERE o.oprkind='l' AND o.oprright = rt.oid
+ AND o.oprresult = result.oid
+ ORDER BY operand""")
+ return result
+
+# list all binary operators
+def list_binary_operator(pgcnx):
+ result = pgcnx.query("""SELECT o.oprname AS binary_op,
+ rt.typname AS right_opr, lt.typname AS left_opr,
+ result.typname AS return_type
+ FROM pg_operator o, pg_type rt, pg_type lt, pg_type result
+ WHERE o.oprkind = 'b' AND o.oprright = rt.oid
+ AND o.oprleft = lt.oid AND o.oprresult = result.oid""")
+ return result
+
+# returns the name, args and return type from all function of lang l
+def list_lang_func(pgcnx, l):
+ result = pgcnx.query("""SELECT p.proname, p.pronargs, t.typname
+ FROM pg_proc p, pg_language l, pg_type t
+ WHERE p.prolang = l.oid AND p.prorettype = t.oid
+ AND l.lanname = '%s'
+ ORDER BY proname""" % l)
+ return result
+
+# lists all the aggregate functions and the type to which they can be applied
+def list_agg_func(pgcnx):
+ result = pgcnx.query("""SELECT p.proname, t.typname
+ FROM pg_aggregate a, pg_proc p, pg_type t
+ WHERE a.aggfnoid = p.oid
+ and p.proargtypes[0] = t.oid
+ ORDER BY proname, typname""")
+ return result
+
+# lists all the operator classes that can be used with each access method as
+# well as the operators that can be used with the respective operator classes
+def list_op_class(pgcnx):
+ result = pgcnx.query("""SELECT am.amname, opc.opcname, opr.oprname
+ FROM pg_am am, pg_amop amop, pg_opclass opc, pg_operator opr
+ WHERE amop.amopid = am.oid and amop.amopclaid = opc.oid
+ AND amop.amopopr = opr.oid order by amname, opcname, oprname""")
+ return result
+
+# demo function - runs all examples
+def demo(pgcnx):
+ import sys, os
+ save_stdout = sys.stdout
+ sys.stdout = os.popen("more", "w")
+ print "Listing simple indices ..."
+ print list_simple_ind(pgcnx)
+ print "Listing all attributes ..."
+ print list_all_attr(pgcnx)
+ print "Listing all user-defined base types ..."
+ print list_user_base_type(pgcnx)
+ print "Listing all left-unary operators defined ..."
+ print list_left_unary_operator(pgcnx)
+ print "Listing all right-unary operators defined ..."
+ print list_right_unary_operator(pgcnx)
+ print "Listing all binary operators ..."
+ print list_binary_operator(pgcnx)
+ print "Listing C external function linked ..."
+ print list_lang_func(pgcnx, 'C')
+ print "Listing C internal functions ..."
+ print list_lang_func(pgcnx, 'internal')
+ print "Listing SQL functions defined ..."
+ print list_lang_func(pgcnx, 'sql')
+ print "Listing 'aggregate functions' ..."
+ print list_agg_func(pgcnx)
+ print "Listing 'operator classes' ..."
+ print list_op_class(pgcnx)
+ del sys.stdout
+ sys.stdout = save_stdout
--- /dev/null
+#!/usr/bin/python
+
+import pg, time
+
+USER="agenor"
+HOST="localhost"
+DB="agenor_fldinfodb"
+TABLE="SOGo_folder_info"
+
+db = pg.DB(DB, HOST, 5432, "", "", USER);
+print "connection:", db
+
+def timeSelect(SELECT, WHERE=None):
+ SQL="SELECT %s FROM %s" % ( SELECT, TABLE )
+ if not WHERE is None: SQL="%s WHERE %s" % ( SQL, WHERE )
+ starttime=time.time()
+ res = db.query(SQL)
+ endtime=time.time()
+ print "perf '%s', %s: %.2fms" % ( SELECT, WHERE, (endtime-starttime)*1000)
+
+timeSelect("COUNT(*)")
+timeSelect("c_tablename, c_dbname")
+timeSelect("c_tablename, c_dbname", "c_foldername='privcal_99827'")
--- /dev/null
+#!/usr/bin/python
+
+import pg, time
+
+USER="agenor"
+HOST="localhost"
+DB="agenor_testhugeperf"
+TABLE="SOGo_huge_quick"
+
+db = pg.DB(DB, HOST, 5432, "", "", USER);
+print "connection:", db
+
+def timeSelect(SELECT, WHERE=None):
+ SQL="SELECT %s FROM %s" % ( SELECT, TABLE )
+ if not WHERE is None: SQL="%s WHERE %s" % ( SQL, WHERE )
+ starttime=time.time()
+ res = db.query(SQL)
+ endtime=time.time()
+ print "perf '%s', %s: %.2fms" % ( SELECT, WHERE, (endtime-starttime)*1000)
+
+timeSelect("COUNT(*)")
+timeSelect("c_pkey, c_sourceid, c_startdate, c_enddate",
+ "c_startdate > 1085068363 AND c_startdate < 1085068373")
+timeSelect("c_sourceid")
--- /dev/null
+#!/usr/bin/python
+
+import pg, time, sys
+
+USER="agenor"
+HOST="localhost"
+DB="agenor_testhugeperf"
+QUICKTABLE="SOGo_huge_quick"
+BLOBTABLE="SOGo_huge_ical"
+
+db = pg.DB(DB, HOST, 5432, "", "", USER);
+print "connection:", db
+
+# index, index, start, end, index,
+QUICKTEMPLATE="""INSERT INTO %s (
+ c_pkey, c_sourceid, c_startdate, c_enddate, c_title, c_attendees,
+ c_isallday, c_sequenceid, c_generation
+) VALUES (
+ %i, 'longsource%iid', %i, %i, 'long title %i',
+ 'hh@skyrix.com,mm@skyrix,je@skyrix.com,fr@skyrix.com,jm@skyrix.com,hh@skyrix.com,mm@skyrix,je@skyrix.com,fr@skyrix.com,jm@skyrix.com,hh@skyrix.com,mm@skyrix,je@skyrix.com,fr@skyrix.com,jm@skyrix.com',
+ 0, 0, 1
+);"""
+
+# index, ical
+ICALTEMPLATE="INSERT INTO %s ( c_pkey, c_data ) VALUES ( %i, '%s' );"
+
+ICALFILE="""BEGIN:VEVENT\r
+DURATION:PT1H\r
+ATTENDEE;CN="Elke Bethke";DIR="addressbook://B156F3F0-9CFD-11D8-8561-000\r
+ D93C1A604:ABPerson":mailto:E.Bethke@Sachsen-Anhalt-Lotto.de\r
+ATTENDEE;CN="Erik Doernenburg";DIR="addressbook://B15FCB0F-9CFD-11D8-8561\r
+ -000D93C1A604:ABPerson":mailto:erik@x101.net\r
+ATTENDEE;CN="Christian Schnelle";DIR="addressbook://B1418D4E-9CFD-11D8-8\r
+ 561-000D93C1A604:ABPerson":mailto:cs@enervation.de\r
+ATTENDEE;CN="Chris Herrenberger";DIR="addressbook://B14A390C-9CFD-11D8-8\r
+ 561-000D93C1A604:ABPerson":invalid:nomail\r
+ATTENDEE;CN="Horst Parplies";DIR="addressbook://B19B47E5-9CFD-11D8-8561-\r
+ 000D93C1A604:ABPerson":mailto:horst.parplies@freenet.de\r
+ATTENDEE;CN="Imdat Solak";DIR="addressbook://B19EDB62-9CFD-11D8-8561-000\r
+ D93C1A604:ABPerson":mailto:imdat@solak.de\r
+ATTENDEE;CN="Jens Enders";DIR="addressbook://B1B6819F-9CFD-11D8-8561-000\r
+ D93C1A604:ABPerson":mailto:jens.enders@skyrix.com\r
+ATTENDEE;CN="Jens Muenster";DIR="addressbook://B1BBA42E-9CFD-11D8-8561-00\r
+ 0D93C1A604:ABPerson":mailto:jens.muenster@skyrix.com\r
+ATTENDEE;CN="Laurent Pierre";DIR="addressbook://9337C270-A825-11D8-B930-\r
+ 000D93C1A604:ABPerson":mailto:laurent.pierre@linagora.com\r
+ATTENDEE;CN="Marcel Weiher";DIR="addressbook://B1F9BB12-9CFD-11D8-8561-0\r
+ 00D93C1A604:ABPerson":mailto:marcel@metaobject.co\r
+DTSTAMP:20040520T140002Z\r
+UID:BD91C454-AA65-11D8-84CA-000D93C1A604\r
+SEQUENCE:3\r
+STATUS:CONFIRMED\r
+DTSTART;TZID=Europe/Berlin:20040618T160000\r
+SUMMARY:SIZE EVENT\r
+X-WR-ITIPSTATUSML:UNCLEAN\r
+END:VEVENT\r
+"""
+
+
+# ******************** INSERT ********************
+
+FROM=int(sys.argv[1])
+TO=FROM+1000000
+#FROM=1
+#TO=1000000
+
+timingstart=time.time()
+
+for i in range(FROM, TO):
+ start=time.time()
+ end=start+(60 * 30)
+
+ QSQL = QUICKTEMPLATE % ( QUICKTABLE, i, i, start, end, i, )
+ BSQL = ICALTEMPLATE % ( BLOBTABLE, i, ICALFILE )
+
+ if i % 10000 == 0:
+ print "%i (%.2fs): quick %s" % ( i, time.time()-timingstart, QSQL )
+ #print "%i: blob %s" % ( i, BSQL )
+ db.query(QSQL + BSQL)
+
--- /dev/null
+#!/usr/bin/python
+
+import pg
+
+USER="agenor"
+HOST="localhost"
+DB="agenor_fldinfodb"
+TABLE="SOGo_folder_info"
+
+db = pg.DB(DB, HOST, 5432, "", "", USER);
+print "connection:", db
+
+for i in range(110001, 250000):
+ SQL=("INSERT INTO %s ( c_foldername, c_tablename, c_dbname ) " + \
+ "VALUES ( 'privcal_%i', 'agenor_tab_%i', 'agenortabledb' );") % \
+ ( TABLE, i, i, )
+ if i % 1000 == 0:
+ print "%i: %s" % ( i, SQL )
+ db.query(SQL)
--- /dev/null
+#!/usr/bin/python
+
+import os, sys
+
+HOST="localhost"
+USER="agenor"
+DB="blah2"
+
+for i in range(920, 10000):
+ DB="agenor%i" % ( i, )
+ res=os.system("createdb -h %s -U %s %s" % ( HOST, USER, DB ))
+ print "%s res: %i" % ( DB, res )
--- /dev/null
+#!/usr/bin/python
+
+import os, sys
+
+HOST="localhost"
+USER="agenor"
+DB="agenortabledb"
+
+for i in range(10000, 60000):
+ NEWTABLE="agenor_tab_%i" % ( i, )
+
+ TABLECREATE="""CREATE TABLE %s
+ ( pkey INT PRIMARY KEY,
+ startdate INT NOT NULL,
+ endate INT NOT NULL,
+ title VARCHAR(1000) NOT NULL,
+ participants VARCHAR(100000) NOT NULL);""" % ( NEWTABLE, )
+
+ CALL="echo '%s' | psql -h %s %s %s" % ( TABLECREATE, HOST, DB, USER )
+ res=os.system(CALL)
+ #res = 1
+ #print "CALL:", CALL
+ print "%s res: %i" % ( NEWTABLE, res )
--- /dev/null
+#!/usr/bin/python
+
+import os, sys
+
+HOST="localhost"
+USER="agenor"
+DB=""
+
+for i in range(1, 10000):
+ NEWUSER="agenor%i" % ( i, )
+ res=os.system("createuser -A -D -h %s -U %s %s" % ( HOST, USER, NEWUSER ))
+ print "%s res: %i" % ( NEWUSER, res )
projects / scalable-opengroupware.org / commitdiff