From 91fcd5480c72b68e044c0c66090094089c0b7e97 Mon Sep 17 00:00:00 2001 From: ML Date: Mon, 5 Nov 2012 19:40:50 +0100 Subject: [PATCH 001/109] Update postgresql/clientparameters.py Alterations made to supportuse in win32 service (forced getuser() to return 'postgres') --- postgresql/clientparameters.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/postgresql/clientparameters.py b/postgresql/clientparameters.py index 18075017..e618694f 100644 --- a/postgresql/clientparameters.py +++ b/postgresql/clientparameters.py @@ -46,12 +46,13 @@ class ClientParameterError(Error): class ServiceDoesNotExistError(ClientParameterError): code = '-*srv' -try: - from getpass import getuser, getpass -except ImportError: - getpass = raw_input - def getuser(): - return 'postgres' +# on win32 when program is run as windows service, getuser falls back to pwn which is not implemented +# therefore it is better to resign from getting default username, which allows service operation + +getpass = raw_input + +def getuser(): + return 'postgres' default_host = 'localhost' default_port = 5432 @@ -141,6 +142,9 @@ def defaults(environ = os.environ): if appdata: pgdata = os.path.join(appdata, pg_appdata_directory) pgpassfile = os.path.join(pgdata, pg_appdata_passfile) + else: + pgdata = '' + pgpassfile = '' else: pgpassfile = os.path.join(userdir, pg_home_passfile) From faedbb7d0b8583f146bbaa00f1ad8160f2c11f1c Mon Sep 17 00:00:00 2001 From: James William Pye Date: Tue, 5 Feb 2013 15:27:40 -0800 Subject: [PATCH 002/109] Broken. Revert "Update postgresql/clientparameters.py" This reverts commit 91fcd5480c72b68e044c0c66090094089c0b7e97. --- postgresql/clientparameters.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/postgresql/clientparameters.py b/postgresql/clientparameters.py index e618694f..18075017 100644 --- a/postgresql/clientparameters.py +++ b/postgresql/clientparameters.py @@ -46,13 +46,12 @@ class ClientParameterError(Error): class ServiceDoesNotExistError(ClientParameterError): code = '-*srv' -# on win32 when program is run as windows service, getuser falls back to pwn which is not implemented -# therefore it is better to resign from getting default username, which allows service operation - -getpass = raw_input - -def getuser(): - return 'postgres' +try: + from getpass import getuser, getpass +except ImportError: + getpass = raw_input + def getuser(): + return 'postgres' default_host = 'localhost' default_port = 5432 @@ -142,9 +141,6 @@ def defaults(environ = os.environ): if appdata: pgdata = os.path.join(appdata, pg_appdata_directory) pgpassfile = os.path.join(pgdata, pg_appdata_passfile) - else: - pgdata = '' - pgpassfile = '' else: pgpassfile = os.path.join(userdir, pg_home_passfile) From 6ef6e1d2df9f128d39ea78ea2a9117e48aca3182 Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Tue, 13 May 2014 18:29:12 -0400 Subject: [PATCH 003/109] Handle unix_socket_directories GUC change in PostgreSQL 9.3 PostgreSQL replaced unix_socket_directory GUC with unix_socket_directories. Handle that in places where this GUC is used. --- postgresql/temporal.py | 11 ++++++++++- postgresql/test/test_cluster.py | 8 +++++++- postgresql/test/test_connect.py | 11 ++++++++++- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/postgresql/temporal.py b/postgresql/temporal.py index 8beb8e05..6e65ee0b 100644 --- a/postgresql/temporal.py +++ b/postgresql/temporal.py @@ -137,8 +137,17 @@ def init(self, listen_addresses = 'localhost', log_destination = 'stderr', log_min_messages = 'FATAL', - unix_socket_directory = cluster.data_directory, )) + + if installation.version_info[:2] < (9, 3): + cluster.settings.update(dict( + unix_socket_directory = cluster.data_directory, + )) + else: + cluster.settings.update(dict( + unix_socket_directories = cluster.data_directory, + )) + cluster.settings.update(dict( max_prepared_transactions = '10', )) diff --git a/postgresql/test/test_cluster.py b/postgresql/test/test_cluster.py index 72748e5a..456279f9 100644 --- a/postgresql/test/test_cluster.py +++ b/postgresql/test/test_cluster.py @@ -29,11 +29,17 @@ def start_cluster(self, logfile = None): def init(self, *args, **kw): self.cluster.init(*args, **kw) + + if self.cluster.installation.version_info[:2] >= (9, 3): + usd = 'unix_socket_directories' + else: + usd = 'unix_socket_directory' + self.cluster.settings.update({ 'max_connections' : '8', 'listen_addresses' : 'localhost', 'port' : '6543', - 'unix_socket_directory' : self.cluster.data_directory, + usd : self.cluster.data_directory, }) def testSilentMode(self): diff --git a/postgresql/test/test_connect.py b/postgresql/test/test_connect.py index cfa48827..93894ef9 100644 --- a/postgresql/test/test_connect.py +++ b/postgresql/test/test_connect.py @@ -86,8 +86,17 @@ def configure_cluster(self): listen_addresses = listen_addresses, log_destination = 'stderr', log_min_messages = 'FATAL', - unix_socket_directory = self.cluster.data_directory, )) + + if self.cluster.installation.version_info[:2] < (9, 3): + self.cluster.settings.update(dict( + unix_socket_directory = self.cluster.data_directory, + )) + else: + self.cluster.settings.update(dict( + unix_socket_directories = self.cluster.data_directory, + )) + # 8.4 turns prepared transactions off by default. if self.cluster.installation.version_info >= (8,1): self.cluster.settings.update(dict( From a186a73089fab3ff7b8e9be08bb0ad9b206fcc0f Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Tue, 13 May 2014 18:32:24 -0400 Subject: [PATCH 004/109] test: Fix SchemaNameError test PostgreSQL 9.3 raises UndefinedTableError instead of SchemaNameError when a reference to a non-existent table in a non-existent schema is made. Fix the test to test for non-existent schema specifically. --- postgresql/test/test_driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/test/test_driver.py b/postgresql/test/test_driver.py index 6d2d58c0..03977351 100644 --- a/postgresql/test/test_driver.py +++ b/postgresql/test/test_driver.py @@ -1355,7 +1355,7 @@ def testSyntaxError(self): @pg_tmp def testSchemaNameError(self): try: - db.prepare("SELECT * FROM sdkfldasjfdskljZknvson.foo")() + db.prepare("CREATE TABLE sdkfldasjfdskljZknvson.foo()")() except pg_exc.SchemaNameError: return self.fail("SchemaNameError was not raised") From 80934b73ab255600c257e74bba4011e87148b9c2 Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Tue, 13 May 2014 18:33:55 -0400 Subject: [PATCH 005/109] test: Fix MessageHook test on PostgreSQL 9.3 PostgreSQL 9.3 no longer raises a NOTICE about automatic creation of an implicit index when PRIMARY KEY is declared (it raises a DEBUG1 instead). Switch the test to an attempt to REINDEX a table without indexes, which is a more consistent NOTICE (this has been a WARNING before PostgreSQL 7.4). --- postgresql/test/test_driver.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/postgresql/test/test_driver.py b/postgresql/test/test_driver.py index 03977351..9c81bab9 100644 --- a/postgresql/test/test_driver.py +++ b/postgresql/test/test_driver.py @@ -1697,10 +1697,11 @@ def testPayloads(self): @pg_tmp def testMessageHook(self): - create = db.prepare('CREATE TEMP TABLE msghook (i INT PRIMARY KEY)') + create = db.prepare('CREATE TEMP TABLE msghook (i INT)') + reindex = db.prepare('REINDEX TABLE msghook') drop = db.prepare('DROP TABLE msghook') parts = [ - create, + reindex, db, db.connector, db.connector.driver, @@ -1716,6 +1717,7 @@ def add(x): for x in parts: x.msghook = add create() + reindex() del x.msghook drop() self.assertEqual(len(notices), len(parts)) From 33cb09d2ea4c72520a2958409e749df4957cf379 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 23 Jul 2015 10:05:14 -0700 Subject: [PATCH 006/109] Refer to local binds in functions and module attributes in module bodies. (Optimize the function a bit) --- postgresql/alock.py | 10 +++++----- postgresql/driver/pq3.py | 14 ++++++++++---- postgresql/pgpassfile.py | 4 ++-- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/postgresql/alock.py b/postgresql/alock.py index c7134de9..cdf413d1 100644 --- a/postgresql/alock.py +++ b/postgresql/alock.py @@ -4,8 +4,8 @@ """ Tools for Advisory Locks """ -from abc import abstractmethod, abstractproperty -from .python.element import Element +import abc +from .python import element __all__ = [ 'ALock', @@ -13,7 +13,7 @@ 'ShareLock', ] -class ALock(Element): +class ALock(element.Element): """ Advisory Lock class for managing the acquisition and release of a sequence of PostgreSQL advisory locks. @@ -32,13 +32,13 @@ def _e_metas(self, ): yield None, headfmt(self.state, self.mode) - @abstractproperty + @abc.abstractproperty def mode(self): """ The mode of the lock class. """ - @abstractproperty + @abc.abstractproperty def __select_statements__(self): """ Implemented by subclasses to return the statements to try, acquire, and diff --git a/postgresql/driver/pq3.py b/postgresql/driver/pq3.py index 40314c0b..a959e46d 100644 --- a/postgresql/driver/pq3.py +++ b/postgresql/driver/pq3.py @@ -1775,11 +1775,17 @@ def _load_copy_chunks(self, chunks, *parameters): def _load_tuple_chunks(self, chunks): pte = self._raise_parameter_tuple_error last = (element.SynchronizeMessage,) + + Bind = element.Bind + Instruction = xact.Instruction + Execute = element.Execute + tuple = tuple + try: for chunk in chunks: bindings = [ ( - element.Bind( + Bind( b'', self._pq_statement_id, self._input_formats, @@ -1788,13 +1794,13 @@ def _load_tuple_chunks(self, chunks): ), (), ), - element.Execute(b'', 1), + Execute(b'', 1), ) for t in chunk ] bindings.append(last) self.database._pq_push( - xact.Instruction( + Instruction( chain.from_iterable(bindings), asynchook = self.database._receive_async ), @@ -2429,7 +2435,7 @@ def _establish(self): # guts of connect() self.pq = None # if any exception occurs past this point, the connection - # will not be usable. + # object will not be usable. timeout = self.connector.connect_timeout sslmode = self.connector.sslmode or 'prefer' failures = [] diff --git a/postgresql/pgpassfile.py b/postgresql/pgpassfile.py index 41589303..e7a505a7 100644 --- a/postgresql/pgpassfile.py +++ b/postgresql/pgpassfile.py @@ -2,7 +2,7 @@ # .pgpassfile - parse and lookup passwords in a pgpassfile ## 'Parse pgpass files and subsequently lookup a password.' -from os.path import exists +import os.path def split(line, len = len): line = line.strip() @@ -54,7 +54,7 @@ def lookup_password_file(path, t): with open(path) as f: return lookup_password(parse(f), t) -def lookup_pgpass(d, passfile, exists = exists): +def lookup_pgpass(d, passfile, exists = os.path.exists): # If the password file exists, lookup the password # using the config's criteria. if exists(passfile): From 2f2df7a3abf06f032b05122677a1937032b51a9c Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Thu, 9 Jun 2016 17:15:42 -0400 Subject: [PATCH 007/109] Add I/O support for jsonb type --- postgresql/test/test_driver.py | 3 +++ postgresql/types/__init__.py | 1 + postgresql/types/io/__init__.py | 4 ++++ postgresql/types/io/stdlib_jsonb.py | 24 ++++++++++++++++++++++++ 4 files changed, 32 insertions(+) create mode 100644 postgresql/types/io/stdlib_jsonb.py diff --git a/postgresql/test/test_driver.py b/postgresql/test/test_driver.py index 9c81bab9..d13c924e 100644 --- a/postgresql/test/test_driver.py +++ b/postgresql/test/test_driver.py @@ -284,6 +284,9 @@ ['00:00:00:00:00:01', '00:00:00:00:00:00', 'ff:ff:ff:ff:ff:ff', '10:00:00:00:00:00'], ], ), + ('jsonb', [ + '{"foo": "bar", "spam": ["ham"]}' + ]) ] try: diff --git a/postgresql/types/__init__.py b/postgresql/types/__init__.py index 63bc03bf..08dd3334 100644 --- a/postgresql/types/__init__.py +++ b/postgresql/types/__init__.py @@ -31,6 +31,7 @@ REGDICTIONARYOID = 3769 JSONOID = 114 +JSONBOID = 3802 XMLOID = 142 MACADDROID = 829 diff --git a/postgresql/types/io/__init__.py b/postgresql/types/io/__init__.py index e05d9f9a..93542be4 100644 --- a/postgresql/types/io/__init__.py +++ b/postgresql/types/io/__init__.py @@ -79,6 +79,10 @@ pg_types.XMLOID, ), + 'stdlib_jsonb' : ( + pg_types.JSONBOID, + ), + # Must be db.typio.identify(contrib_hstore = 'hstore')'d 'contrib_hstore' : ( 'contrib_hstore', diff --git a/postgresql/types/io/stdlib_jsonb.py b/postgresql/types/io/stdlib_jsonb.py new file mode 100644 index 00000000..08223569 --- /dev/null +++ b/postgresql/types/io/stdlib_jsonb.py @@ -0,0 +1,24 @@ +from ...types import JSONBOID + + +def jsonb_pack(x, typeio): + jsonb = typeio.encode(x) + return b'\x01' + jsonb + + +def jsonb_unpack(x, typeio): + if x[0] != 1: + raise ValueError('unexpected JSONB format version: {!r}'.format(x[0])) + return typeio.decode(x[1:]) + + +def _jsonb_io_factory(oid, typeio): + _pack = lambda x: jsonb_pack(x, typeio) + _unpack = lambda x: jsonb_unpack(x, typeio) + + return (_pack, _unpack, str) + + +oid_to_io = { + JSONBOID: _jsonb_io_factory +} From 1f61818171e5170c78981580c832bdd2dbca50e9 Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Wed, 22 Jun 2016 10:58:01 -0400 Subject: [PATCH 008/109] Fix PendingDeprecationWarning in numeric_convert_digits --- postgresql/types/io/stdlib_decimal.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/postgresql/types/io/stdlib_decimal.py b/postgresql/types/io/stdlib_decimal.py index a61c7824..3deee388 100644 --- a/postgresql/types/io/stdlib_decimal.py +++ b/postgresql/types/io/stdlib_decimal.py @@ -130,13 +130,17 @@ def numeric_pack(x, def numeric_convert_digits(d, str = str, int = int): i = iter(d) - for x in str(next(i)): - # no leading zeros - yield int(x) - # leading digit should not include zeros - for y in i: - for x in str(y).rjust(4, '0'): + try: + for x in str(next(i)): + # no leading zeros yield int(x) + # leading digit should not include zeros + for y in i: + for x in str(y).rjust(4, '0'): + yield int(x) + except StopIteration: + # Python 3.5+ does not like generators raising StopIteration + return numeric_signs = { numeric_negative : 1, From 0dadc44c778361b5e0b0479196f442eeccb6a524 Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Wed, 22 Jun 2016 11:01:20 -0400 Subject: [PATCH 009/109] Bump minimum required Python version to 3.3 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 0c273772..cba1b1d4 100755 --- a/setup.py +++ b/setup.py @@ -5,9 +5,9 @@ import sys import os -if sys.version_info[:2] < (3,1): +if sys.version_info[:2] < (3,3): sys.stderr.write( - "ERROR: py-postgresql is for Python 3.1 and greater." + os.linesep + "ERROR: py-postgresql is for Python 3.3 and greater." + os.linesep ) sys.stderr.write( "HINT: setup.py was ran using Python " + \ From 8c770ae36d47039fd251dfe90f4b2ff71bd28964 Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Wed, 22 Jun 2016 11:04:28 -0400 Subject: [PATCH 010/109] Stamp v1.2.0 --- postgresql/documentation/changes-v1.2.rst | 9 +++++++++ postgresql/documentation/index.rst | 1 + postgresql/project.py | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 postgresql/documentation/changes-v1.2.rst diff --git a/postgresql/documentation/changes-v1.2.rst b/postgresql/documentation/changes-v1.2.rst new file mode 100644 index 00000000..55642010 --- /dev/null +++ b/postgresql/documentation/changes-v1.2.rst @@ -0,0 +1,9 @@ +Changes in v1.2 +=============== + +1.2.0 released on 2016-06-23 +---------------------------- + + * PostgreSQL 9.3 compatibility fixes (Elvis) + * Python 3.5 compatibility fixes (Elvis) + * Add support for JSONB type (Elvis) diff --git a/postgresql/documentation/index.rst b/postgresql/documentation/index.rst index 66cece8a..322438d4 100644 --- a/postgresql/documentation/index.rst +++ b/postgresql/documentation/index.rst @@ -37,6 +37,7 @@ Changes .. toctree:: :maxdepth: 1 + changes-v1.2 changes-v1.1 changes-v1.0 diff --git a/postgresql/project.py b/postgresql/project.py index aec21dab..13c84b2a 100644 --- a/postgresql/project.py +++ b/postgresql/project.py @@ -10,5 +10,5 @@ contact = 'python-general@pgfoundry.org' abstract = 'Driver and tools library for PostgreSQL' -version_info = (1, 1, 0) +version_info = (1, 2, 0) version = '.'.join(map(str, version_info)) From 96989784f385c81880e32e748408cbddcd142e35 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Mon, 27 Jun 2016 10:55:00 -0700 Subject: [PATCH 011/109] Python 3.5 doesn't like localizing builtins inside functions, apparently. --- postgresql/driver/pq3.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/postgresql/driver/pq3.py b/postgresql/driver/pq3.py index a959e46d..5ba7426e 100644 --- a/postgresql/driver/pq3.py +++ b/postgresql/driver/pq3.py @@ -1772,14 +1772,13 @@ def _load_copy_chunks(self, chunks, *parameters): self.database._pq_complete() self.database.pq.synchronize() - def _load_tuple_chunks(self, chunks): + def _load_tuple_chunks(self, chunks, tuple=tuple): pte = self._raise_parameter_tuple_error last = (element.SynchronizeMessage,) Bind = element.Bind Instruction = xact.Instruction Execute = element.Execute - tuple = tuple try: for chunk in chunks: From 59be98dcd4c1aa71b9b2798a19c3b51ba9cd2f27 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Mon, 5 Sep 2016 23:43:46 -0700 Subject: [PATCH 012/109] Add mapping entry for JSON names. --- postgresql/project.py | 2 +- postgresql/types/__init__.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/postgresql/project.py b/postgresql/project.py index 13c84b2a..f848e6f0 100644 --- a/postgresql/project.py +++ b/postgresql/project.py @@ -10,5 +10,5 @@ contact = 'python-general@pgfoundry.org' abstract = 'Driver and tools library for PostgreSQL' -version_info = (1, 2, 0) +version_info = (1, 2, 1) version = '.'.join(map(str, version_info)) diff --git a/postgresql/types/__init__.py b/postgresql/types/__init__.py index 08dd3334..6481d929 100644 --- a/postgresql/types/__init__.py +++ b/postgresql/types/__init__.py @@ -158,6 +158,8 @@ REGDICTIONARYOID : 'regdictionary', XMLOID : 'xml', + JSONOID : 'json', + JSONBOID : 'jsonb', MACADDROID : 'macaddr', INETOID : 'inet', From 33465976b275e92236d1d4a3c38e3c5639deca9d Mon Sep 17 00:00:00 2001 From: James William Pye Date: Tue, 6 Sep 2016 00:55:23 -0700 Subject: [PATCH 013/109] Use the exact range rather than adjusting each time. --- postgresql/driver/pq3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/postgresql/driver/pq3.py b/postgresql/driver/pq3.py index 5ba7426e..e4810cb3 100644 --- a/postgresql/driver/pq3.py +++ b/postgresql/driver/pq3.py @@ -1916,8 +1916,8 @@ def __init__(self, ident, database, description = ()): proctup['_proid'], # ($1::type, $2::type, ... $n::type) ', '.join([ - '$%d::%s' %(x + 1, database.typio.sql_type_from_oid(proargs[x])) - for x in range(len(proargs)) + '$%d::%s' %(x, database.typio.sql_type_from_oid(proargs[x])) + for x in range(1, len(proargs)+1) ]), # Description for anonymous record returns (description and \ From f7d07581f3cfb9e5b4fb4749dc019139813bdc93 Mon Sep 17 00:00:00 2001 From: Lukasz Walach Date: Tue, 2 May 2017 16:31:45 +0300 Subject: [PATCH 014/109] Use setuptools in favour of distutils.core Fallback to distutils.core if setuptools aren't available --- postgresql/release/distutils.py | 5 ++++- setup.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/postgresql/release/distutils.py b/postgresql/release/distutils.py index dc93709d..d875aeff 100644 --- a/postgresql/release/distutils.py +++ b/postgresql/release/distutils.py @@ -13,7 +13,10 @@ import sys import os from ..project import version, name, identity as url -from distutils.core import Extension, Command +try: + from setuptools import Extension, Command +except ImportError as e: + from distutils.core import Extension, Command LONG_DESCRIPTION = """ py-postgresql is a set of Python modules providing interfaces to various parts diff --git a/setup.py b/setup.py index cba1b1d4..4ba72288 100755 --- a/setup.py +++ b/setup.py @@ -25,5 +25,8 @@ sys.dont_write_bytecode = False if __name__ == '__main__': - from distutils.core import setup + try: + from setuptools import setup + except ImportError as e: + from distutils.core import setup setup(**defaults) From d4a5d57a6a841d904a0b2dc9a8de26e525bc8dd8 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 8 Jun 2018 23:32:36 -0700 Subject: [PATCH 015/109] Remove note about the previous project name --- postgresql/release/distutils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/postgresql/release/distutils.py b/postgresql/release/distutils.py index d875aeff..491b2db1 100644 --- a/postgresql/release/distutils.py +++ b/postgresql/release/distutils.py @@ -56,14 +56,6 @@ If a successful connection is made to the remote host, it will provide a Python console with the database connection bound to the `db` name. - - -History -------- - -py-postgresql is not yet another PostgreSQL driver, it's been in development for -years. py-postgresql is the Python 3 port of the ``pg_proboscis`` driver and -integration of the other ``pg/python`` projects. """ CLASSIFIERS = [ From 22c6295488aab242d60ca6f5c830c488b713a46e Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 8 Jun 2018 23:36:08 -0700 Subject: [PATCH 016/109] Remove title and the obvious --- LICENSE | 7 ------- 1 file changed, 7 deletions(-) diff --git a/LICENSE b/LICENSE index daa35b8e..875a834e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,10 +1,3 @@ -BSD Licensed Software - - Unless stated otherwise, the contained software is - copyright 2004-2009, James Williame Pye. - For more information: http://python.projects.postgresql.org - - Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: From ef04b3cf234521befa5d27e6204e338a557bed18 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sat, 9 Jun 2018 00:01:49 -0700 Subject: [PATCH 017/109] Normalize doc-string formatting --- postgresql/protocol/element3.py | 103 ++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 26 deletions(-) diff --git a/postgresql/protocol/element3.py b/postgresql/protocol/element3.py index e4ccf4fd..39fca329 100644 --- a/postgresql/protocol/element3.py +++ b/postgresql/protocol/element3.py @@ -1,7 +1,9 @@ ## # .protocol.element3 ## -'PQ version 3.0 elements' +""" +PQ version 3.0 elements. +""" import sys import os import pprint @@ -300,7 +302,9 @@ class Suspension(EmptyMessage): Suspension.SingleInstance = SuspensionMessage class Ready(Message): - 'Ready for new query' + """ + Ready for new query message. + """ type = message_types[b'Z'[0]] possible_states = ( message_types[b'I'[0]], @@ -319,7 +323,7 @@ def serialize(self): class Notice(Message, dict): """ - Notification message + Notification message. Used by PQ to emit INFO, NOTICE, and WARNING messages among other severities. @@ -353,7 +357,9 @@ def parse(self): raise RuntimeError("cannot parse ClientNotice") class Error(Notice): - """Incoming error""" + """ + Error information message. + """ type = message_types[b'E'[0]] __slots__ = () @@ -368,7 +374,9 @@ def parse(self): raise RuntimeError("cannot serialize ClientError") class FunctionResult(Message): - """Function result value""" + """ + Function result value. + """ type = message_types[b'V'[0]] __slots__ = ('result',) @@ -394,7 +402,9 @@ def parse(typ, data): return typ(data) class AttributeTypes(TupleMessage): - """Tuple attribute types""" + """ + Tuple attribute types. + """ type = message_types[b't'[0]] __slots__ = () @@ -410,7 +420,9 @@ def parse(typ, data): return typ(unpack('!%dL'%(ac,), args)) class TupleDescriptor(TupleMessage): - """Tuple description""" + """ + Tuple structure description. + """ type = message_types[b'T'[0]] struct = Struct("!LhLhlh") __slots__ = () @@ -442,7 +454,9 @@ def parse(typ, data): return typ(atts) class Tuple(TupleMessage): - """Incoming tuple""" + """ + Tuple Data. + """ type = message_types[b'D'[0]] __slots__ = () @@ -480,7 +494,9 @@ def parse(typ, data, pass class KillInformation(Message): - 'Backend cancellation information' + """ + Backend cancellation information. + """ type = message_types[b'K'[0]] struct = Struct("!LL") __slots__ = ('pid', 'key') @@ -497,7 +513,9 @@ def parse(typ, data): return typ(*typ.struct.unpack(data)) class CancelRequest(KillInformation): - 'Abort the query in the specified backend' + """ + Abort the query in the specified backend. + """ type = b'' from .version import CancelRequestCode as version packed_version = version.bytes() @@ -519,7 +537,9 @@ def parse(typ, data): return typ(*typ.struct.unpack(data[4:])) class NegotiateSSL(Message): - "Discover backend's SSL support" + """ + Discover backend's SSL support. + """ type = b'' from .version import NegotiateSSLCode as version packed_version = version.bytes() @@ -605,7 +625,9 @@ def parse(typ, data): } class Authentication(Message): - """Authentication(request, salt)""" + """ + Authentication(request, salt) + """ type = message_types[b'R'[0]] __slots__ = ('request', 'salt') @@ -621,38 +643,50 @@ def parse(typ, data): return typ(ulong_unpack(data[0:4]), data[4:]) class Password(StringMessage): - 'Password supplement' + """ + Password supplement. + """ type = message_types[b'p'[0]] __slots__ = ('data',) class Disconnect(EmptyMessage): - 'Close the connection' + """ + Connection closed message. + """ type = message_types[b'X'[0]] __slots__ = () DisconnectMessage = Message.__new__(Disconnect) Disconnect.SingleInstance = DisconnectMessage class Flush(EmptyMessage): - 'Flush' + """ + Flush message. + """ type = message_types[b'H'[0]] __slots__ = () FlushMessage = Message.__new__(Flush) Flush.SingleInstance = FlushMessage class Synchronize(EmptyMessage): - 'Synchronize' + """ + Synchronize. + """ type = message_types[b'S'[0]] __slots__ = () SynchronizeMessage = Message.__new__(Synchronize) Synchronize.SingleInstance = SynchronizeMessage class Query(StringMessage): - """Execute the query with the given arguments""" + """ + Execute the query with the given arguments. + """ type = message_types[b'Q'[0]] __slots__ = ('data',) class Parse(Message): - """Parse a query with the specified argument types""" + """ + Parse a query with the specified argument types. + """ type = message_types[b'P'[0]] __slots__ = ('name', 'statement', 'argtypes') @@ -741,7 +775,9 @@ def parse(typ, message_data): return typ(name, statement, aformats, args, rformats) class Execute(Message): - """Fetch results from the specified Portal""" + """ + Fetch results from the specified Portal. + """ type = message_types[b'E'[0]] __slots__ = ('name', 'max') @@ -758,7 +794,9 @@ def parse(typ, data): return typ(name, ulong_unpack(max)) class Describe(StringMessage): - """Describe a Portal or Prepared Statement""" + """ + Request a description of a Portal or Prepared Statement. + """ type = message_types[b'D'[0]] __slots__ = ('data',) @@ -784,7 +822,9 @@ class DescribePortal(Describe): __slots__ = ('data',) class Close(StringMessage): - """Generic Close""" + """ + Generic Close + """ type = message_types[b'C'[0]] __slots__ = () @@ -802,17 +842,24 @@ def parse(typ, data): return super().parse(data[1:]) class CloseStatement(Close): - """Close the specified Statement""" + """ + Close the specified Statement + """ subtype = message_types[b'S'[0]] __slots__ = () class ClosePortal(Close): - """Close the specified Portal""" + """ + Close the specified Portal + """ subtype = message_types[b'P'[0]] __slots__ = () class Function(Message): - """Execute the specified function with the given arguments""" + """ + Execute the specified function with the given arguments + """ + type = message_types[b'F'[0]] __slots__ = ('oid', 'aformats', 'arguments', 'rformat') @@ -881,12 +928,16 @@ def parse(typ, data): ]) class CopyToBegin(CopyBegin): - """Begin copying to""" + """ + Begin copying to. + """ type = message_types[b'H'[0]] __slots__ = ('format', 'formats') class CopyFromBegin(CopyBegin): - """Begin copying from""" + """ + Begin copying from. + """ type = message_types[b'G'[0]] __slots__ = ('format', 'formats') From 90d8feffa1dfed3e9f0d949d6343c278c695bd22 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sat, 9 Jun 2018 00:02:25 -0700 Subject: [PATCH 018/109] Remove whitespace --- postgresql/protocol/xact3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/protocol/xact3.py b/postgresql/protocol/xact3.py index 8064b618..d8e35650 100644 --- a/postgresql/protocol/xact3.py +++ b/postgresql/protocol/xact3.py @@ -232,7 +232,7 @@ def state_machine(self): ) )) return - else: + else: self.authok = self.authtype # Done authenticating, pick up the killinfo and the ready message. From e43e729b5b149acfdc531d90fbd02c1cf3cd9660 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 20 Jul 2018 21:20:49 -0700 Subject: [PATCH 019/109] Add rtd config for python3 --- readthedocs.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 readthedocs.yml diff --git a/readthedocs.yml b/readthedocs.yml new file mode 100644 index 00000000..ba184c23 --- /dev/null +++ b/readthedocs.yml @@ -0,0 +1,5 @@ +build: + image: latest + +python: + version: 3.6 From d9bc00bbbb6722db88bf23fabeba9b8c989dd6a3 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 20 Jul 2018 21:40:39 -0700 Subject: [PATCH 020/109] Clearly intelligent --- postgresql/documentation/sphinx/admin.rst | 1 + postgresql/documentation/sphinx/alock.rst | 1 + postgresql/documentation/sphinx/bin.rst | 1 + postgresql/documentation/sphinx/changes-v1.0.rst | 1 + postgresql/documentation/sphinx/changes-v1.1.rst | 1 + postgresql/documentation/sphinx/changes-v1.2.rst | 1 + postgresql/documentation/sphinx/clientparameters.rst | 1 + postgresql/documentation/sphinx/cluster.rst | 1 + postgresql/documentation/sphinx/copyman.rst | 1 + postgresql/documentation/sphinx/driver.rst | 1 + postgresql/documentation/sphinx/gotchas.rst | 1 + postgresql/documentation/sphinx/index.rst | 1 + postgresql/documentation/sphinx/lib.rst | 1 + postgresql/documentation/sphinx/notifyman.rst | 1 + postgresql/documentation/sphinx/reference.rst | 1 + 15 files changed, 15 insertions(+) create mode 120000 postgresql/documentation/sphinx/admin.rst create mode 120000 postgresql/documentation/sphinx/alock.rst create mode 120000 postgresql/documentation/sphinx/bin.rst create mode 120000 postgresql/documentation/sphinx/changes-v1.0.rst create mode 120000 postgresql/documentation/sphinx/changes-v1.1.rst create mode 120000 postgresql/documentation/sphinx/changes-v1.2.rst create mode 120000 postgresql/documentation/sphinx/clientparameters.rst create mode 120000 postgresql/documentation/sphinx/cluster.rst create mode 120000 postgresql/documentation/sphinx/copyman.rst create mode 120000 postgresql/documentation/sphinx/driver.rst create mode 120000 postgresql/documentation/sphinx/gotchas.rst create mode 120000 postgresql/documentation/sphinx/index.rst create mode 120000 postgresql/documentation/sphinx/lib.rst create mode 120000 postgresql/documentation/sphinx/notifyman.rst create mode 120000 postgresql/documentation/sphinx/reference.rst diff --git a/postgresql/documentation/sphinx/admin.rst b/postgresql/documentation/sphinx/admin.rst new file mode 120000 index 00000000..b5ba2b75 --- /dev/null +++ b/postgresql/documentation/sphinx/admin.rst @@ -0,0 +1 @@ +../admin.rst \ No newline at end of file diff --git a/postgresql/documentation/sphinx/alock.rst b/postgresql/documentation/sphinx/alock.rst new file mode 120000 index 00000000..fc663467 --- /dev/null +++ b/postgresql/documentation/sphinx/alock.rst @@ -0,0 +1 @@ +../alock.rst \ No newline at end of file diff --git a/postgresql/documentation/sphinx/bin.rst b/postgresql/documentation/sphinx/bin.rst new file mode 120000 index 00000000..b2f59715 --- /dev/null +++ b/postgresql/documentation/sphinx/bin.rst @@ -0,0 +1 @@ +../bin.rst \ No newline at end of file diff --git a/postgresql/documentation/sphinx/changes-v1.0.rst b/postgresql/documentation/sphinx/changes-v1.0.rst new file mode 120000 index 00000000..6cb2068b --- /dev/null +++ b/postgresql/documentation/sphinx/changes-v1.0.rst @@ -0,0 +1 @@ +../changes-v1.0.rst \ No newline at end of file diff --git a/postgresql/documentation/sphinx/changes-v1.1.rst b/postgresql/documentation/sphinx/changes-v1.1.rst new file mode 120000 index 00000000..598bf89d --- /dev/null +++ b/postgresql/documentation/sphinx/changes-v1.1.rst @@ -0,0 +1 @@ +../changes-v1.1.rst \ No newline at end of file diff --git a/postgresql/documentation/sphinx/changes-v1.2.rst b/postgresql/documentation/sphinx/changes-v1.2.rst new file mode 120000 index 00000000..08b7164d --- /dev/null +++ b/postgresql/documentation/sphinx/changes-v1.2.rst @@ -0,0 +1 @@ +../changes-v1.2.rst \ No newline at end of file diff --git a/postgresql/documentation/sphinx/clientparameters.rst b/postgresql/documentation/sphinx/clientparameters.rst new file mode 120000 index 00000000..49e862b4 --- /dev/null +++ b/postgresql/documentation/sphinx/clientparameters.rst @@ -0,0 +1 @@ +../clientparameters.rst \ No newline at end of file diff --git a/postgresql/documentation/sphinx/cluster.rst b/postgresql/documentation/sphinx/cluster.rst new file mode 120000 index 00000000..cafbe524 --- /dev/null +++ b/postgresql/documentation/sphinx/cluster.rst @@ -0,0 +1 @@ +../cluster.rst \ No newline at end of file diff --git a/postgresql/documentation/sphinx/copyman.rst b/postgresql/documentation/sphinx/copyman.rst new file mode 120000 index 00000000..53623d4f --- /dev/null +++ b/postgresql/documentation/sphinx/copyman.rst @@ -0,0 +1 @@ +../copyman.rst \ No newline at end of file diff --git a/postgresql/documentation/sphinx/driver.rst b/postgresql/documentation/sphinx/driver.rst new file mode 120000 index 00000000..fbcf3238 --- /dev/null +++ b/postgresql/documentation/sphinx/driver.rst @@ -0,0 +1 @@ +../driver.rst \ No newline at end of file diff --git a/postgresql/documentation/sphinx/gotchas.rst b/postgresql/documentation/sphinx/gotchas.rst new file mode 120000 index 00000000..52aefb0b --- /dev/null +++ b/postgresql/documentation/sphinx/gotchas.rst @@ -0,0 +1 @@ +../gotchas.rst \ No newline at end of file diff --git a/postgresql/documentation/sphinx/index.rst b/postgresql/documentation/sphinx/index.rst new file mode 120000 index 00000000..ba7f001c --- /dev/null +++ b/postgresql/documentation/sphinx/index.rst @@ -0,0 +1 @@ +../index.rst \ No newline at end of file diff --git a/postgresql/documentation/sphinx/lib.rst b/postgresql/documentation/sphinx/lib.rst new file mode 120000 index 00000000..b1e774a8 --- /dev/null +++ b/postgresql/documentation/sphinx/lib.rst @@ -0,0 +1 @@ +../lib.rst \ No newline at end of file diff --git a/postgresql/documentation/sphinx/notifyman.rst b/postgresql/documentation/sphinx/notifyman.rst new file mode 120000 index 00000000..4741bfd6 --- /dev/null +++ b/postgresql/documentation/sphinx/notifyman.rst @@ -0,0 +1 @@ +../notifyman.rst \ No newline at end of file diff --git a/postgresql/documentation/sphinx/reference.rst b/postgresql/documentation/sphinx/reference.rst new file mode 120000 index 00000000..38d3cd51 --- /dev/null +++ b/postgresql/documentation/sphinx/reference.rst @@ -0,0 +1 @@ +../reference.rst \ No newline at end of file From 422ddb831f082433fadc8e907bed0eeac0c741af Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 20 Jul 2018 22:00:16 -0700 Subject: [PATCH 021/109] Recommend against use --- postgresql/documentation/lib.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/postgresql/documentation/lib.rst b/postgresql/documentation/lib.rst index 6db3811a..592b96fa 100644 --- a/postgresql/documentation/lib.rst +++ b/postgresql/documentation/lib.rst @@ -2,11 +2,9 @@ Categories and Libraries ************************ This chapter discusses the usage and implementation of connection categories and -libraries. - -.. note:: - First-time users are encouraged to read the `Audience and Motivation`_ - section first. +libraries. Originally these features were written with general purpose use in mind; +however, it is recommended that these features **not** be used in applications. +They are primarily used internally by the the driver and may be removed in the future. Libraries are a collection of SQL statements that can be bound to a connection. Libraries are *normally* bound directly to the connection object as From fb5f9702a36700e71dd666546bfac7dae173f3e7 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sat, 21 Jul 2018 13:36:51 -0700 Subject: [PATCH 022/109] Update links and add recommendation for asyncpg --- README | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README b/README index 1270af30..38791d99 100644 --- a/README +++ b/README @@ -1,10 +1,16 @@ About ===== -py-postgresql is a Python 3 package providing modules to work with PostgreSQL. +py-postgresql is a Python 3 package providing modules for working with PostgreSQL. This includes a high-level driver, and many other tools that support a developer working with PostgreSQL databases. +For a high performance async interface, MagicStack's +`asyncpg ` should be considered. + +py-postgresql, currently, does not have direct support for high-level async +interfaces provided by recent versions of Python. Future versions may change this. + Installation ------------ @@ -35,7 +41,7 @@ Further Information Online documentation can be retrieved from: - http://python.projects.postgresql.org + http://py-postgresql.readthedocs.io Or, you can read them in your pager: python -m postgresql.documentation.index From 70d763a1b540d9c607d70324053267fa2c88d7c5 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sat, 21 Jul 2018 13:39:15 -0700 Subject: [PATCH 023/109] Don't bother with rst --- README | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README b/README index 38791d99..184ade65 100644 --- a/README +++ b/README @@ -5,8 +5,8 @@ py-postgresql is a Python 3 package providing modules for working with PostgreSQ This includes a high-level driver, and many other tools that support a developer working with PostgreSQL databases. -For a high performance async interface, MagicStack's -`asyncpg ` should be considered. +For a high performance async interface, MagicStack'sasyncpg +http://github.com/MagicStack/asyncpg should be considered. py-postgresql, currently, does not have direct support for high-level async interfaces provided by recent versions of Python. Future versions may change this. From fdf086ce4649482585e1274bf6d1d8a6ac00ef83 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sat, 21 Jul 2018 13:40:11 -0700 Subject: [PATCH 024/109] Formatting --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index 184ade65..80b6f21d 100644 --- a/README +++ b/README @@ -5,7 +5,7 @@ py-postgresql is a Python 3 package providing modules for working with PostgreSQ This includes a high-level driver, and many other tools that support a developer working with PostgreSQL databases. -For a high performance async interface, MagicStack'sasyncpg +For a high performance async interface, MagicStack's asyncpg http://github.com/MagicStack/asyncpg should be considered. py-postgresql, currently, does not have direct support for high-level async From 91ac4229ea752692c802cf681466dcd6c04f0385 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 2 Aug 2018 22:44:13 -0700 Subject: [PATCH 025/109] Use consistent type --- postgresql/port/_optimized/buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/port/_optimized/buffer.c b/postgresql/port/_optimized/buffer.c index 29a1ece3..0b6cf2eb 100644 --- a/postgresql/port/_optimized/buffer.c +++ b/postgresql/port/_optimized/buffer.c @@ -100,7 +100,7 @@ p_new(PyTypeObject *subtype, PyObject *args, PyObject *kw) static char p_at_least(struct p_place *p, uint32_t amount) { - int32_t current = 0; + uint32_t current = 0; struct p_list *pl; pl = p->list; From d08145cabb8b70889e6e482aac59fb35b5421d72 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 2 Aug 2018 22:47:46 -0700 Subject: [PATCH 026/109] Don't include HTML; prefer rtd use or local sphinx builds --- MANIFEST.in | 1 - 1 file changed, 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 855b6c57..fcb59ede 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,4 +4,3 @@ recursive-include postgresql *.c recursive-include postgresql *.sql recursive-include postgresql *.txt recursive-include postgresql/documentation/sphinx *.rst conf.py -recursive-include postgresql/documentation/html * From 4495bb58ebb2e65c70f7e3420e6f1683862641c9 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 2 Aug 2018 22:48:53 -0700 Subject: [PATCH 027/109] Update IRIs and contact information. --- postgresql/__init__.py | 10 ++++------ postgresql/project.py | 10 ++++------ postgresql/release/distutils.py | 8 ++++---- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/postgresql/__init__.py b/postgresql/__init__.py index b96607d0..60ce8e54 100644 --- a/postgresql/__init__.py +++ b/postgresql/__init__.py @@ -1,15 +1,13 @@ ## # py-postgresql root package -# http://python.projects.postgresql.org +# http://github.com/python-postgres/fe ## """ py-postgresql is a Python package for using PostgreSQL. This includes low-level -protocol tools, a driver(PG-API and DB-API), and cluster management tools. +protocol tools, a driver(PG-API and DB-API 2.0), and cluster management tools. -If it's not documented in the narratives, `postgresql.documentation.index`, then -the stability of the APIs should *not* be trusted. - -See for more information about PostgreSQL. +See for more information about PostgreSQL and +for information about Python. """ __all__ = [ '__author__', diff --git a/postgresql/project.py b/postgresql/project.py index f848e6f0..395b3f62 100644 --- a/postgresql/project.py +++ b/postgresql/project.py @@ -1,13 +1,11 @@ -'project information' +""" +Project information. +""" -#: project name name = 'py-postgresql' - -#: IRI based project identity -identity = 'http://python.projects.postgresql.org/' +identity = 'http://github.com/python-postgres/fe' meaculpa = 'Python+Postgres' -contact = 'python-general@pgfoundry.org' abstract = 'Driver and tools library for PostgreSQL' version_info = (1, 2, 1) diff --git a/postgresql/release/distutils.py b/postgresql/release/distutils.py index 491b2db1..c896a047 100644 --- a/postgresql/release/distutils.py +++ b/postgresql/release/distutils.py @@ -20,10 +20,10 @@ LONG_DESCRIPTION = """ py-postgresql is a set of Python modules providing interfaces to various parts -of PostgreSQL. Notably, it provides a pure-Python driver + C optimizations for +of PostgreSQL. Primarily, it provides a pure-Python driver with some C optimizations for querying a PostgreSQL database. -http://python.projects.postgresql.org +http://github.com/python-postgres/fe Features: @@ -160,9 +160,9 @@ def standard_setup_keywords(build_extensions = True, prefix = default_prefix): 'description' : 'PostgreSQL driver and tools library.', 'long_description' : LONG_DESCRIPTION, 'author' : 'James William Pye', - 'author_email' : 'x@jwp.name', + 'author_email' : 'james.pye@gmail.com', 'maintainer' : 'James William Pye', - 'maintainer_email' : 'python-general@pgfoundry.org', + 'maintainer_email' : 'james.pye@gmail.com', 'url' : url, 'classifiers' : CLASSIFIERS, 'packages' : list(prefixed_packages(prefix = prefix)), From ef7b9a96ac39fdd17a6cea8b13a58646ac754218 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 2 Aug 2018 23:29:42 -0700 Subject: [PATCH 028/109] Rewrite doc-annotations as sphinx parameters --- postgresql/clientparameters.py | 45 +++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/postgresql/clientparameters.py b/postgresql/clientparameters.py index 18075017..59ed876b 100644 --- a/postgresql/clientparameters.py +++ b/postgresql/clientparameters.py @@ -154,7 +154,7 @@ def defaults(environ = os.environ): if os.path.exists(v): yield (k,), v -def envvars(environ = os.environ, modifier : "environment variable key modifier" = 'PG'.__add__): +def envvars(environ = os.environ, modifier = 'PG'.__add__): """ Create a clientparams dictionary from the given environment variables. @@ -182,6 +182,8 @@ def envvars(environ = os.environ, modifier : "environment variable key modifier" The 'PG' prefix can be customized via the `modifier` argument. However, PGSYSCONFDIR will not respect any such change as it's not a client parameter itself. + + :param modifier: environment variable key modifier """ hostaddr = modifier('HOSTADDR') reqssl = modifier('REQUIRESSL') @@ -274,7 +276,9 @@ def append_db_client_parameters(option, opt_str, value, parser): ) def append_settings(option, opt_str, value, parser): - 'split the string into a (key,value) pair tuple' + """ + split the string into a (key,value) pair tuple + """ kv = value.split('=', 1) if len(kv) != 2: raise OptionValueError("invalid setting argument, %r" %(value,)) @@ -367,11 +371,7 @@ class DefaultParser(StandardParser): """ standard_option_list = default_optparse_options -def resolve_password( - parameters : "a fully normalized set of client parameters(dict)", - getpass = getpass, - prompt_title = '', -): +def resolve_password(parameters, getpass = getpass, prompt_title = ''): """ Given a parameters dictionary, resolve the 'password' key. @@ -386,6 +386,8 @@ def resolve_password( Finally, remove the pgpassfile key as the password has been resolved for the given parameters. + + :param parameters: a fully normalized set of client parameters(dict) """ prompt_for_password = parameters.pop('prompt_password', False) pgpassfile = parameters.pop('pgpassfile', None) @@ -570,7 +572,7 @@ def normalize(iter): def resolve_pg_service_file( environ = os.environ, default_pg_sysconfdir = None, - default_pg_service_filename = pg_service_filename + default_pg_service_filename = pg_service_filename ): sysconfdir = environ.get(pg_sysconfdir_envvar, default_pg_sysconfdir) if sysconfdir: @@ -578,18 +580,27 @@ def resolve_pg_service_file( return None def collect( - parsed_options : "options parsed using the `DefaultParser`" = None, - no_defaults : "Don't build-out defaults like 'user' from getpass.getuser()" = False, - environ : "environment variables to use, `None` to disable" = os.environ, - environ_prefix : "prefix to use for collecting environment variables" = 'PG', - default_pg_sysconfdir : "default 'PGSYSCONFDIR' to use" = None, - pg_service_file : "the pg-service file to actually use" = None, - prompt_title : "additional title to use if a prompt request is made" = '', - parameters : "base-client parameters to use(applied after defaults)" = (), -): + parsed_options = None, + no_defaults = False, + environ = os.environ, + environ_prefix = 'PG', + default_pg_sysconfdir = None, + pg_service_file = None, + prompt_title = '', + parameters = (), + ): """ Build a normalized client parameters dictionary for use with a connection construction interface. + + :param parsed_options: options parsed using the `DefaultParser` + :param no_defaults: Don't build-out defaults like 'user' from getpass.getuser() + :param environ: environment variables to use, `None` to disable + :param environ_prefix: prefix to use for collecting environment variables + :param default_pg_sysconfdir: default 'PGSYSCONFDIR' to use + :param pg_service_file: the pg-service file to actually use + :param prompt_title: additional title to use if a prompt request is made + :param parameters: base-client parameters to use(applied after defaults) """ d_parameters = [] d_parameters.append([('config-environ', environ)]) From eef4bc2bb86b8956e366044dc34267c3502276cb Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sun, 9 Feb 2020 16:30:42 -0700 Subject: [PATCH 029/109] Revert commit broken from 33465976b275e9, fixes #89. --- postgresql/driver/pq3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/postgresql/driver/pq3.py b/postgresql/driver/pq3.py index e4810cb3..b4695af5 100644 --- a/postgresql/driver/pq3.py +++ b/postgresql/driver/pq3.py @@ -1916,8 +1916,8 @@ def __init__(self, ident, database, description = ()): proctup['_proid'], # ($1::type, $2::type, ... $n::type) ', '.join([ - '$%d::%s' %(x, database.typio.sql_type_from_oid(proargs[x])) - for x in range(1, len(proargs)+1) + '$%d::%s' %(x + 1, database.typio.sql_type_from_oid(proargs[x])) + for x in range(0, len(proargs)) ]), # Description for anonymous record returns (description and \ From 2509badb90a23c7c8b85c146f960bcc7bb8d57aa Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sun, 9 Feb 2020 17:08:04 -0700 Subject: [PATCH 030/109] Remove string annotation (likely incomprehensible to mypy) and minor doc-string formatting. --- postgresql/protocol/version.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/postgresql/protocol/version.py b/postgresql/protocol/version.py index fd76995c..53b592d8 100644 --- a/postgresql/protocol/version.py +++ b/postgresql/protocol/version.py @@ -1,19 +1,22 @@ ## # .protocol.version ## -'PQ version class' +""" +PQ version class used by startup messages. +""" from struct import Struct version_struct = Struct('!HH') class Version(tuple): - """Version((major, minor)) -> Version + """ + Version((major, minor)) -> Version Version serializer and parser. """ major = property(fget = lambda s: s[0]) minor = property(fget = lambda s: s[1]) - def __new__(subtype, major_minor : '(major, minor)'): + def __new__(subtype, major_minor): (major, minor) = major_minor major = int(major) minor = int(minor) From 8e3c2297414c368f9210353a1611a310b5c76b7d Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sun, 9 Feb 2020 17:32:51 -0700 Subject: [PATCH 031/109] Correct yml indentation. --- readthedocs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs.yml b/readthedocs.yml index ba184c23..d53ffa68 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -1,5 +1,5 @@ build: - image: latest + image: latest python: - version: 3.6 + version: 3.6 From 7ee74cd9118fd0b2c4fd0b2864b7e1c2506be519 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sun, 9 Feb 2020 17:46:11 -0700 Subject: [PATCH 032/109] For tests, explicitly disable replication and conditionally perform check for crypt. Tests are executed against databases in clusters that have restrained resources. Replication being enabled was triggering failures with the current configuration. Having the crypt user unconditionally created was causing errors in more recent versions of PostgreSQL. Only check crypt with now unsupported versions. --- postgresql/temporal.py | 10 +++--- postgresql/test/test_cluster.py | 6 +++- postgresql/test/test_connect.py | 60 ++++++++++++++++++--------------- 3 files changed, 44 insertions(+), 32 deletions(-) diff --git a/postgresql/temporal.py b/postgresql/temporal.py index 6e65ee0b..2db48f13 100644 --- a/postgresql/temporal.py +++ b/postgresql/temporal.py @@ -110,6 +110,7 @@ def init(self, 'could not find the default pg_config', details = inshint ) + vi = installation.version_info cluster = Cluster(installation, self.cluster_path,) # If it exists already, destroy it. @@ -130,6 +131,10 @@ def init(self, creator = cluster ) + if vi[:2] > (9,6): + # Default changed in 10.x + cluster.settings['max_wal_senders'] = '0' + cluster.settings.update(dict( port = str(self.cluster_port), max_connections = '20', @@ -137,6 +142,7 @@ def init(self, listen_addresses = 'localhost', log_destination = 'stderr', log_min_messages = 'FATAL', + max_prepared_transactions = '10', )) if installation.version_info[:2] < (9, 3): @@ -148,10 +154,6 @@ def init(self, unix_socket_directories = cluster.data_directory, )) - cluster.settings.update(dict( - max_prepared_transactions = '10', - )) - # Start it up. with open(self.logfile, 'w') as lfo: cluster.start(logfile = lfo) diff --git a/postgresql/test/test_cluster.py b/postgresql/test/test_cluster.py index 456279f9..4f781f6c 100644 --- a/postgresql/test/test_cluster.py +++ b/postgresql/test/test_cluster.py @@ -30,11 +30,15 @@ def start_cluster(self, logfile = None): def init(self, *args, **kw): self.cluster.init(*args, **kw) - if self.cluster.installation.version_info[:2] >= (9, 3): + vi = self.cluster.installation.version_info[:2] + if vi >= (9, 3): usd = 'unix_socket_directories' else: usd = 'unix_socket_directory' + if vi > (9, 6): + self.cluster.settings['max_wal_senders'] = '0' + self.cluster.settings.update({ 'max_connections' : '8', 'listen_addresses' : 'localhost', diff --git a/postgresql/test/test_connect.py b/postgresql/test/test_connect.py index 93894ef9..88902c04 100644 --- a/postgresql/test/test_connect.py +++ b/postgresql/test/test_connect.py @@ -51,7 +51,7 @@ def __init__(self, *args, **kw): super().__init__(*args, **kw) self.installation = installation.default() self.cluster_path = \ - 'py_unittest_pg_cluster_' \ + 'pypg_test_' \ + str(os.getpid()) + getattr(self, 'cluster_path_suffix', '') if self.installation is None: @@ -68,6 +68,8 @@ def __init__(self, *args, **kw): if self.cluster.initialized(): self.cluster.drop() + self.disable_replication = self.installation.version_info[:2] > (9, 6) + def configure_cluster(self): self.cluster_port = find_available_port() if self.cluster_port is None: @@ -79,6 +81,7 @@ def configure_cluster(self): listen_addresses = '127.0.0.1' if has_ipv6: listen_addresses += ',::1' + self.cluster.settings.update(dict( port = str(self.cluster_port), max_connections = '6', @@ -88,6 +91,11 @@ def configure_cluster(self): log_min_messages = 'FATAL', )) + if self.disable_replication: + self.cluster.settings.update({ + 'max_wal_senders': '0', + }) + if self.cluster.installation.version_info[:2] < (9, 3): self.cluster.settings.update(dict( unix_socket_directory = self.cluster.data_directory, @@ -157,23 +165,35 @@ class test_connect(TestCaseWithCluster): params = {} cluster_path_suffix = '_test_connect' + mk_common_users = """ + CREATE USER md5 WITH ENCRYPTED PASSWORD 'md5_password'; + CREATE USER password WITH ENCRYPTED PASSWORD 'password_password'; + CREATE USER trusted; + """ + + mk_crypt_user = """ + -- crypt doesn't work with encrypted passwords: + -- http://www.postgresql.org/docs/8.2/interactive/auth-methods.html#AUTH-PASSWORD + CREATE USER crypt WITH UNENCRYPTED PASSWORD 'crypt_password'; + """ + def __init__(self, *args, **kw): super().__init__(*args,**kw) # 8.4 nixed this. - self.do_crypt = self.cluster.installation.version_info < (8,4) + vi = self.cluster.installation.version_info + self.check_crypt_user = (vi < (8,4)) def configure_cluster(self): super().configure_cluster() - self.cluster.settings.update({ - 'log_min_messages' : 'log', - }) + self.cluster.settings['log_min_messages'] = 'log' # Configure the hba file with the supported methods. with open(self.cluster.hba_file, 'w') as hba: hosts = ['0.0.0.0/0',] if has_ipv6: hosts.append('0::0/0') - methods = ['md5', 'password'] + (['crypt'] if self.do_crypt else []) + + methods = ['md5', 'password'] + (['crypt'] if self.check_crypt_user else []) for h in hosts: for m in methods: # user and method are the same name. @@ -181,6 +201,7 @@ def configure_cluster(self): h = h, m = m )]) + # trusted hba.writelines(["local all all trust\n"]) hba.writelines(["host test trusted 0.0.0.0/0 trust\n"]) @@ -193,26 +214,11 @@ def configure_cluster(self): def initialize_database(self): super().initialize_database() + with self.cluster.connection(user = 'test') as db: - db.execute( - """ -CREATE USER md5 WITH - ENCRYPTED PASSWORD 'md5_password' -; - --- crypt doesn't work with encrypted passwords: --- http://www.postgresql.org/docs/8.2/interactive/auth-methods.html#AUTH-PASSWORD -CREATE USER crypt WITH - UNENCRYPTED PASSWORD 'crypt_password' -; - -CREATE USER password WITH - ENCRYPTED PASSWORD 'password_password' -; - -CREATE USER trusted; - """ - ) + db.execute(self.mk_common_users) + if self.check_crypt_user: + db.execute(self.mk_crypt_user) def test_pg_open_SQL_ASCII(self): # postgresql.open @@ -364,7 +370,7 @@ def test_dbapi_connect(self): MD5.cursor().execute, 'select 1' ) - if self.do_crypt: + if self.check_crypt_user: CRYPT = dbapi20.connect( user = 'crypt', database = 'test', @@ -449,7 +455,7 @@ def test_md5_connect(self): self.assertEqual(c.prepare('select current_user').first(), 'md5') def test_crypt_connect(self): - if self.do_crypt: + if self.check_crypt_user: c = self.cluster.connection( user = 'crypt', password = 'crypt_password', From 23f5927f5ff3bb28c48f51a231ec008e69ff9297 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sun, 9 Feb 2020 19:55:14 -0700 Subject: [PATCH 033/109] Modify builtins using the imported module as pypy was complaining. fixes #97 --- postgresql/temporal.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/postgresql/temporal.py b/postgresql/temporal.py index 2db48f13..21096a24 100644 --- a/postgresql/temporal.py +++ b/postgresql/temporal.py @@ -6,6 +6,7 @@ """ import os import atexit +import builtins from collections import deque from .cluster import Cluster, ClusterError from . import installation @@ -179,7 +180,7 @@ def new_pg_tmp_connection(l = extras, c = c, sbid = 'sandbox' + str(self.sandbox return l[-1] # The new builtins. - builtins = { + local_builtins = { 'db' : c, 'prepare' : c.prepare, 'xact' : c.xact, @@ -193,37 +194,37 @@ def new_pg_tmp_connection(l = extras, c = c, sbid = 'sandbox' + str(self.sandbox if not self.builtins_stack: # Store any of those set or not set. current = { - k : __builtins__[k] for k in self.builtins_keys - if k in __builtins__ + k : builtins.__dict__[k] for k in self.builtins_keys + if k in builtins.__dict__ } self.builtins_stack.append((current, [])) # Store and push. - self.builtins_stack.append((builtins, extras)) - __builtins__.update(builtins) + self.builtins_stack.append((local_builtins, extras)) + builtins.__dict__.update(local_builtins) self.sandbox_id += 1 def pop(self, exc, drop_schema = 'DROP SCHEMA sandbox{0} CASCADE'.format): - builtins, extras = self.builtins_stack.pop() + local_builtins, extras = self.builtins_stack.pop() self.sandbox_id -= 1 # restore __builtins__ if len(self.builtins_stack) > 1: - __builtins__.update(self.builtins_stack[-1][0]) + builtins.__dict__.update(self.builtins_stack[-1][0]) else: previous = self.builtins_stack.popleft() for x in self.builtins_keys: if x in previous: - __builtins__[x] = previous[x] + builtins.__dict__[x] = previous[x] else: # Wasn't set before. - __builtins__.pop(x, None) + builtins.__dict__.pop(x, None) # close popped connection, but only if we're not in an interrupt. # However, temporal will always terminate all backends atexit. if exc is None or isinstance(exc, Exception): # Interrupt then close. Just in case something is lingering. - for xdb in [builtins['db']] + list(extras): + for xdb in [local_builtins['db']] + list(extras): if xdb.closed is False: # In order for a clean close of the connection, # interrupt before closing. It is still From 39ca19843c30d679addf44a13a40314961afb6a9 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 17 Sep 2020 18:12:08 -0700 Subject: [PATCH 034/109] Correct oversight in IPv6 host IRIs; redundant bracket stripping. --- postgresql/iri.py | 2 +- postgresql/test/test_iri.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/postgresql/iri.py b/postgresql/iri.py index 30e75596..5e635dfa 100644 --- a/postgresql/iri.py +++ b/postgresql/iri.py @@ -46,7 +46,7 @@ def structure(d, fieldproc = ri.unescape): if host.startswith('unix:'): cpd['unix'] = host[len('unix:'):].replace(':','/') else: - cpd['host'] = host[1:-1] + cpd['host'] = host else: cpd['host'] = fieldproc(host) diff --git a/postgresql/test/test_iri.py b/postgresql/test/test_iri.py index 7792438c..cb6a0abc 100644 --- a/postgresql/test/test_iri.py +++ b/postgresql/test/test_iri.py @@ -84,6 +84,22 @@ ] class test_iri(unittest.TestCase): + def testIP6Hosts(self): + """ + Validate that IPv6 hosts are properly extracted. + """ + s = [ + ('pq://[::1]/db', '::1'), + ('pq://[::1]:1234/db', '::1'), + ('pq://[1:2:3::1]/db', '1:2:3::1'), + ('pq://[1:2:3::1]:1234/db', '1:2:3::1'), + ('pq://[]:1234/db', ''), + ('pq://[]/db', ''), + ] + for i, h in s: + p = pg_iri.parse(i) + self.assertEqual(p['host'], h) + def testPresentPasswordObscure(self): "password is present in IRI, and obscure it" s = 'pq://user:pass@host:port/dbname' From 6606b2e71872959887640f487117c9685865e53f Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 17 Sep 2020 21:03:57 -0700 Subject: [PATCH 035/109] Avoid working harder than necessary and remove the annotations-as-documentation strings. --- postgresql/python/socket.py | 50 +++++++++++++------------------------ 1 file changed, 17 insertions(+), 33 deletions(-) diff --git a/postgresql/python/socket.py b/postgresql/python/socket.py index 0c8b6730..6cdffdca 100644 --- a/postgresql/python/socket.py +++ b/postgresql/python/socket.py @@ -5,7 +5,6 @@ import os import random import socket -import math import errno import ssl @@ -51,7 +50,9 @@ def fatal_exception_message(typ, err) -> (str, None): return getattr(err, 'strerror', '') def secure(self, socket : socket.socket) -> ssl.SSLSocket: - "secure a socket with SSL" + """ + Secure a socket with SSL. + """ if self.socket_secure is not None: return ssl.wrap_socket(socket, **self.socket_secure) else: @@ -69,9 +70,9 @@ def __call__(self, timeout = None): return s def __init__(self, - socket_create : "positional parameters given to socket.socket()", - socket_connect : "parameter given to socket.connect()", - socket_secure : "keywords given to ssl.wrap_socket" = None, + socket_create, + socket_connect, + socket_secure = None, ): self.socket_create = socket_create self.socket_connect = socket_connect @@ -81,36 +82,19 @@ def __str__(self): return 'socket' + repr(self.socket_connect) def find_available_port( - interface : "attempt to bind to interface" = 'localhost', - address_family : "address family to use (default: AF_INET)" = socket.AF_INET, - limit : "Number tries to make before giving up" = 1024, - port_range = (6600, 56600) -) -> (int, None): + interface = 'localhost', + address_family = socket.AF_INET, +): """ Find an available port on the given interface for the given address family. - - Returns a port number that was successfully bound to or `None` if the - attempt limit was reached. """ - i = 0 - while i < limit: - i += 1 - port = ( - math.floor( - random.random() * (port_range[1] - port_range[0]) - ) + port_range[0] - ) - s = socket.socket(address_family, socket.SOCK_STREAM,) - try: - s.bind(('localhost', port)) - s.close() - except socket.error as e: - s.close() - if e.errno in (errno.EACCES, errno.EADDRINUSE, errno.EINTR): - # try again - continue - break - else: - port = None + + port = None + s = socket.socket(address_family, socket.SOCK_STREAM,) + try: + s.bind(('localhost', 0)) + port = s.getsockname()[1] + finally: + s.close() return port From d271b1007b0185525d9dcf96eaa5ffa4569cecde Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 17 Sep 2020 21:11:21 -0700 Subject: [PATCH 036/109] Remove more non-typing annotations. --- postgresql/cluster.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/postgresql/cluster.py b/postgresql/cluster.py index 5ceb18b3..a1d4b31c 100644 --- a/postgresql/cluster.py +++ b/postgresql/cluster.py @@ -155,8 +155,8 @@ def hba_file(self, join = os.path.join): ) def __init__(self, - installation : "installation object", - data_directory : "path to the data directory", + installation, + data_directory, ): self.installation = installation self.data_directory = os.path.abspath(data_directory) @@ -191,9 +191,7 @@ def __exit__(self, typ, val, tb): self.wait_until_stopped() def init(self, - password : \ - "Password to assign to the " \ - "cluster's superuser(`user` keyword)." = None, + password = None, **kw ): """ @@ -334,8 +332,8 @@ def drop(self): os.rmdir(self.data_directory) def start(self, - logfile : "Where to send stderr" = None, - settings : "Mapping of runtime parameters" = None + logfile = None, + settings = None ): """ Start the cluster. @@ -573,8 +571,8 @@ def ready_for_connections(self): return e if e is not None else True def wait_until_started(self, - timeout : "how long to wait before throwing a timeout exception" = 10, - delay : "how long to sleep before re-testing" = 0.05, + timeout = 10, + delay = 0.05, ): """ After the `start` method is used, this can be ran in order to block @@ -625,8 +623,8 @@ def wait_until_started(self, time.sleep(delay) def wait_until_stopped(self, - timeout : "how long to wait before throwing a timeout exception" = 10, - delay : "how long to sleep before re-testing" = 0.05 + timeout = 10, + delay = 0.05 ): """ After the `stop` method is used, this can be ran in order to block until From de3895da1b533c3cc79e91b4b46bff959fccfe25 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 17 Sep 2020 21:41:02 -0700 Subject: [PATCH 037/109] Attempt to be a responsible subprocess user(use .communicate). --- postgresql/cluster.py | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/postgresql/cluster.py b/postgresql/cluster.py index a1d4b31c..2b3f8ac3 100644 --- a/postgresql/cluster.py +++ b/postgresql/cluster.py @@ -192,6 +192,7 @@ def __exit__(self, typ, val, tb): def init(self, password = None, + timeout = None, **kw ): """ @@ -253,29 +254,24 @@ def init(self, cmd, close_fds = close_fds, bufsize = 1024 * 5, # not expecting this to ever be filled. - stdin = sp.PIPE, + stdin = None, stdout = logfile, # stderr is used to identify a reasonable error message. stderr = sp.PIPE, ) - # stdin is not used; it is not desirable for initdb to be attached. - p.stdin.close() - while True: - try: - rc = p.wait() - break - except OSError as e: - if e.errno != errno.EINTR: - raise - finally: - if p.stdout is not None: - p.stdout.close() + try: + stdout, stderr = p.communicate(timeout=timeout) + except sp.TimeoutExpired: + p.kill() + stdout, stderr = p.communicate() + finally: + rc = p.returncode if rc != 0: # initdb returned non-zero, pickup stderr and attach to exception. - r = p.stderr.read().strip() + r = stderr try: msg = r.decode('utf-8') except UnicodeDecodeError: @@ -283,6 +279,7 @@ def init(self, msg = os.linesep.join([ repr(x)[2:-1] for x in r.splitlines() ]) + raise InitDBError( "initdb exited with non-zero status", details = { @@ -293,11 +290,6 @@ def init(self, creator = self ) finally: - if p is not None: - for x in (p.stderr, p.stdin, p.stdout): - if x is not None: - x.close() - if supw_tmp is not None: n = supw_tmp.name supw_tmp.close() From 01bcc9acf9c48406e61da041fe96ea5b4d5f052d Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 17 Sep 2020 21:43:05 -0700 Subject: [PATCH 038/109] Use text as the dummy target; reltime does not appear to exist in 12.4. --- postgresql/test/test_driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/test/test_driver.py b/postgresql/test/test_driver.py index d13c924e..62740e46 100644 --- a/postgresql/test/test_driver.py +++ b/postgresql/test/test_driver.py @@ -1812,7 +1812,7 @@ class test_typio(unittest.TestCase): @pg_tmp def testIdentify(self): # It just exercises the code path. - db.typio.identify(contrib_hstore = 'pg_catalog.reltime') + db.typio.identify(contrib_hstore = 'pg_catalog.text') @pg_tmp def testArrayNulls(self): From 3a093f36232663cdb68e6daff3ffdcd857ff979f Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 17 Sep 2020 21:45:09 -0700 Subject: [PATCH 039/109] Use the exception chain to communicate unlikely system-level failures. --- postgresql/temporal.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/postgresql/temporal.py b/postgresql/temporal.py index 21096a24..1c128bea 100644 --- a/postgresql/temporal.py +++ b/postgresql/temporal.py @@ -124,9 +124,10 @@ def init(self, logfile = None, ) - # Configure - self.cluster_port = find_available_port() - if self.cluster_port is None: + try: + self.cluster_port = find_available_port() + except: + # Rely on chain. raise ClusterError( 'could not find a port for the test cluster on localhost', creator = cluster From 176b0c7ef30d2ec02fa9b6f1fc57e66bd1541c60 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 17 Sep 2020 21:52:18 -0700 Subject: [PATCH 040/109] Remove old e-mail and comment regarding imported parts. --- AUTHORS | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/AUTHORS b/AUTHORS index 0373c8b8..58c8986d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,5 +1,5 @@ Contributors: - James William Pye [faults are mostly mine] + James William Pye [faults are mostly mine] Elvis Pranskevichus William Grzybowski [subjective paramstyle] Barry Grussling [inet/cidr support] @@ -8,13 +8,8 @@ Contributors: Support by Donation: AppCove Network -Further Credits -=============== - -When licenses match, people win. Code is occasionally imported from other -projects to enhance py-postgresql and to allow users to enjoy dependency free -installation. - +Imported +======== DB-API 2.0 Test Case -------------------- From 25e21b29a5f1adc1b6f0fa4e733ecefaa045b497 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 18 Sep 2020 11:32:01 -0700 Subject: [PATCH 041/109] Reveal the appropriate exception during dbapi20.connect(); db.close chained ClientCannotConnect. --- postgresql/driver/dbapi20.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/postgresql/driver/dbapi20.py b/postgresql/driver/dbapi20.py index 6436f334..500e44c4 100644 --- a/postgresql/driver/dbapi20.py +++ b/postgresql/driver/dbapi20.py @@ -323,6 +323,7 @@ class Connection(Connection): Warning DatabaseError = DatabaseError NotSupportedError = NotSupportedError + _dbapi_connected = False def autocommit_set(self, val): if val: @@ -354,9 +355,10 @@ def connect(self, *args, **kw): super().connect(*args, **kw) self._xact = self.xact() self._xact.start() + self._dbapi_connected = True def close(self): - if self.closed: + if self.closed and self._dbapi_connected: raise Error( "connection already closed", source = 'CLIENT', From 05ee2b083527346d5d1fc1d18003bd67df2fe365 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 18 Sep 2020 11:32:55 -0700 Subject: [PATCH 042/109] Document the issue with host/port defaults and endpoints. --- postgresql/driver/dbapi20.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/postgresql/driver/dbapi20.py b/postgresql/driver/dbapi20.py index 500e44c4..4f6e30b8 100644 --- a/postgresql/driver/dbapi20.py +++ b/postgresql/driver/dbapi20.py @@ -401,6 +401,9 @@ def rollback(self): def connect(**kw): """ Create a DB-API connection using the given parameters. + + Due to the way defaults are populated, when connecting to a local filesystem socket + using the `unix` keyword parameter, `host` and `port` must also be set to ``None``. """ std_params = pg_param.collect(prompt_title = None) params = pg_param.normalize( From db2f0c1f9ba9e316e90fe25a2b92ce0ef07a0e8f Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 18 Sep 2020 11:45:43 -0700 Subject: [PATCH 043/109] Turn the connected flag on if the close passes so that subsequent closes will raise the desired exception. --- postgresql/driver/dbapi20.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/postgresql/driver/dbapi20.py b/postgresql/driver/dbapi20.py index 4f6e30b8..f03cfe4f 100644 --- a/postgresql/driver/dbapi20.py +++ b/postgresql/driver/dbapi20.py @@ -323,7 +323,7 @@ class Connection(Connection): Warning DatabaseError = DatabaseError NotSupportedError = NotSupportedError - _dbapi_connected = False + _dbapi_connected_flag = False def autocommit_set(self, val): if val: @@ -355,15 +355,16 @@ def connect(self, *args, **kw): super().connect(*args, **kw) self._xact = self.xact() self._xact.start() - self._dbapi_connected = True + self._dbapi_connected_flag = True def close(self): - if self.closed and self._dbapi_connected: + if self.closed and self._dbapi_connected_flag: raise Error( "connection already closed", source = 'CLIENT', creator = self ) + self._dbapi_connected_flag = True super().close() def cursor(self): From 031653663fec40589b467e0db645dd0c6278bee5 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 18 Sep 2020 14:22:20 -0700 Subject: [PATCH 044/109] Bump version and include a warning regarding the corrected DB-API connect exception. --- README | 11 +++++++++++ postgresql/documentation/changes-v1.2.rst | 9 +++++++++ postgresql/project.py | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/README b/README index 80b6f21d..20815803 100644 --- a/README +++ b/README @@ -11,6 +11,17 @@ http://github.com/MagicStack/asyncpg should be considered. py-postgresql, currently, does not have direct support for high-level async interfaces provided by recent versions of Python. Future versions may change this. +Errata +------ + +.. warning:: + `postgresql.driver.dbapi20.connect` will now raise `ClientCannotConnectError` directly. + Exception traps around connect should still function, but the `__context__` attribute + on the error instance will be `None` in the usual failure case as it is no longer + incorrectly chained. Trapping `ClientCannotConnectError` ahead of `Error` should + allow both cases to co-exist in the event that data is being extracted from + the `ClientCannotConnectError`. + Installation ------------ diff --git a/postgresql/documentation/changes-v1.2.rst b/postgresql/documentation/changes-v1.2.rst index 55642010..363b1557 100644 --- a/postgresql/documentation/changes-v1.2.rst +++ b/postgresql/documentation/changes-v1.2.rst @@ -1,6 +1,15 @@ Changes in v1.2 =============== +1.2.2 released on 2020-09-17 +---------------------------- + + * Correct broken Connection.proc. + * Correct IPv6 IRI host oversight. + * Document an ambiguity case of DB-API 2.0 connection creation and the workaround(unix vs host/port). + * DB-API 2.0 connect() failures caused an undesired exception chain; ClientCannotConnect is now raised. + * Minor maintenance on tests and support modules. + 1.2.0 released on 2016-06-23 ---------------------------- diff --git a/postgresql/project.py b/postgresql/project.py index 395b3f62..f7c4f7bd 100644 --- a/postgresql/project.py +++ b/postgresql/project.py @@ -8,5 +8,5 @@ meaculpa = 'Python+Postgres' abstract = 'Driver and tools library for PostgreSQL' -version_info = (1, 2, 1) +version_info = (1, 2, 2) version = '.'.join(map(str, version_info)) From e8db04c149c6262087ec43026f7f509a2f67505a Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 18 Sep 2020 14:32:13 -0700 Subject: [PATCH 045/109] include a warning regarding the corrected DB-API connect exception. --- postgresql/release/distutils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/postgresql/release/distutils.py b/postgresql/release/distutils.py index c896a047..c887905d 100644 --- a/postgresql/release/distutils.py +++ b/postgresql/release/distutils.py @@ -19,6 +19,14 @@ from distutils.core import Extension, Command LONG_DESCRIPTION = """ +.. warning:: + `postgresql.driver.dbapi20.connect` will now raise `ClientCannotConnectError` directly. + Exception traps around connect should still function, but the `__context__` attribute + on the error instance will be `None` in the usual failure case as it is no longer + incorrectly chained. Trapping `ClientCannotConnectError` ahead of `Error` should + allow both cases to co-exist in the event that data is being extracted from + the `ClientCannotConnectError`. + py-postgresql is a set of Python modules providing interfaces to various parts of PostgreSQL. Primarily, it provides a pure-Python driver with some C optimizations for querying a PostgreSQL database. From 957f8290ee08af4bc4affad221a6b8dab342fb0a Mon Sep 17 00:00:00 2001 From: James William Pye Date: Tue, 22 Sep 2020 11:21:15 -0700 Subject: [PATCH 046/109] Remove trailing spaces. --- postgresql/test/test_dbapi20.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/postgresql/test/test_dbapi20.py b/postgresql/test/test_dbapi20.py index feba8d58..3cbd6626 100644 --- a/postgresql/test/test_dbapi20.py +++ b/postgresql/test/test_dbapi20.py @@ -54,8 +54,8 @@ class test_dbapi20(unittest.TestCase): """ Test a database self.driver for DB API 2.0 compatibility. This implementation tests Gadfly, but the TestCase - is structured so that other self.drivers can subclass this - test case to ensure compiliance with the DB-API. It is + is structured so that other self.drivers can subclass this + test case to ensure compiliance with the DB-API. It is expected that this TestCase may be expanded in the future if ambiguities or edge conditions are discovered. @@ -65,9 +65,9 @@ class test_dbapi20(unittest.TestCase): self.driver, connect_args and connect_kw_args. Class specification should be as follows: - import dbapi20 + import dbapi20 class mytest(dbapi20.DatabaseAPI20Test): - [...] + [...] __rcs_id__ = 'Id: dbapi20.py,v 1.10 2003/10/09 03:14:14 zenzen Exp' __version__ = 'Revision: 1.10' @@ -98,10 +98,10 @@ def tearDown(self): try: cur = con.cursor() for ddl in (self.xddl1, self.xddl2): - try: + try: cur.execute(ddl) con.commit() - except self.driver.Error: + except self.driver.Error: # Assume table didn't exist. Other tests will check if # execute is busted. pass From 3f7bc2c3b6fdae79e961e34926fe58f7e9123aae Mon Sep 17 00:00:00 2001 From: James William Pye Date: Tue, 22 Sep 2020 11:40:21 -0700 Subject: [PATCH 047/109] Hold the DB-API 2.0 exception change until 1.3 for fear of breaking offline dependencies and add a test checking for desired behaviour. --- README | 2 +- postgresql/driver/dbapi20.py | 5 ++++- postgresql/release/distutils.py | 2 +- postgresql/test/test_connect.py | 11 +++++++++++ 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/README b/README index 20815803..bfb53949 100644 --- a/README +++ b/README @@ -15,7 +15,7 @@ Errata ------ .. warning:: - `postgresql.driver.dbapi20.connect` will now raise `ClientCannotConnectError` directly. + In v1.3, `postgresql.driver.dbapi20.connect` will now raise `ClientCannotConnectError` directly. Exception traps around connect should still function, but the `__context__` attribute on the error instance will be `None` in the usual failure case as it is no longer incorrectly chained. Trapping `ClientCannotConnectError` ahead of `Error` should diff --git a/postgresql/driver/dbapi20.py b/postgresql/driver/dbapi20.py index f03cfe4f..d0a2ac6b 100644 --- a/postgresql/driver/dbapi20.py +++ b/postgresql/driver/dbapi20.py @@ -323,6 +323,9 @@ class Connection(Connection): Warning DatabaseError = DatabaseError NotSupportedError = NotSupportedError + + # Explicitly manage DB-API connected state to properly + # throw the already closed error. This will be active in 1.3. _dbapi_connected_flag = False def autocommit_set(self, val): @@ -358,7 +361,7 @@ def connect(self, *args, **kw): self._dbapi_connected_flag = True def close(self): - if self.closed and self._dbapi_connected_flag: + if self.closed:# and self._dbapi_connected_flag: raise Error( "connection already closed", source = 'CLIENT', diff --git a/postgresql/release/distutils.py b/postgresql/release/distutils.py index c887905d..920b822c 100644 --- a/postgresql/release/distutils.py +++ b/postgresql/release/distutils.py @@ -20,7 +20,7 @@ LONG_DESCRIPTION = """ .. warning:: - `postgresql.driver.dbapi20.connect` will now raise `ClientCannotConnectError` directly. + In v1.3, `postgresql.driver.dbapi20.connect` will now raise `ClientCannotConnectError` directly. Exception traps around connect should still function, but the `__context__` attribute on the error instance will be `None` in the usual failure case as it is no longer incorrectly chained. Trapping `ClientCannotConnectError` ahead of `Error` should diff --git a/postgresql/test/test_connect.py b/postgresql/test/test_connect.py index 88902c04..4e29b956 100644 --- a/postgresql/test/test_connect.py +++ b/postgresql/test/test_connect.py @@ -410,6 +410,17 @@ def test_dbapi_connect(self): TRUST.cursor().execute, 'select 1' ) + def test_dbapi_connect_failure(self): + host, port = self.cluster.address() + badlogin = (lambda: dbapi20.connect( + user = '--', + database = '--', + password = '...', + host = host, port = port, + **self.params + )) + self.assertRaises(pg_exc.ClientCannotConnectError, badlogin) + def test_IP4_connect(self): C = pg_driver.default.ip4( user = 'test', From 2a3f4ef20c7d6c142fe81afac9b0e8a63a5fab19 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Tue, 22 Sep 2020 11:54:34 -0700 Subject: [PATCH 048/109] Adjust changes to reflect the pending status of the exception correction. --- postgresql/documentation/changes-v1.2.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/postgresql/documentation/changes-v1.2.rst b/postgresql/documentation/changes-v1.2.rst index 363b1557..8b79fa16 100644 --- a/postgresql/documentation/changes-v1.2.rst +++ b/postgresql/documentation/changes-v1.2.rst @@ -1,13 +1,13 @@ Changes in v1.2 =============== -1.2.2 released on 2020-09-17 +1.2.2 released on 2020-09-22 ---------------------------- * Correct broken Connection.proc. * Correct IPv6 IRI host oversight. * Document an ambiguity case of DB-API 2.0 connection creation and the workaround(unix vs host/port). - * DB-API 2.0 connect() failures caused an undesired exception chain; ClientCannotConnect is now raised. + * (Pending, active in 1.3) DB-API 2.0 connect() failures caused an undesired exception chain; ClientCannotConnect is now raised. * Minor maintenance on tests and support modules. 1.2.0 released on 2016-06-23 From 41902eaf95fbc2dded842898b19288f4f8ba3af1 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Tue, 22 Sep 2020 12:40:10 -0700 Subject: [PATCH 049/109] Add long description media type. --- postgresql/release/distutils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/postgresql/release/distutils.py b/postgresql/release/distutils.py index 920b822c..e921b7b1 100644 --- a/postgresql/release/distutils.py +++ b/postgresql/release/distutils.py @@ -167,6 +167,7 @@ def standard_setup_keywords(build_extensions = True, prefix = default_prefix): 'version' : version, 'description' : 'PostgreSQL driver and tools library.', 'long_description' : LONG_DESCRIPTION, + 'long_description_content_type' : 'text/x-rst', 'author' : 'James William Pye', 'author_email' : 'james.pye@gmail.com', 'maintainer' : 'James William Pye', From 19697ab6ce7ec5153c32e89aed9c633af2284c33 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 24 Sep 2020 10:29:55 -0700 Subject: [PATCH 050/109] Reduce number of checks as it's no longer doing the random allocation attempts; also this was clogging windows. --- postgresql/test/test_python.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/postgresql/test/test_python.py b/postgresql/test/test_python.py index 16d66cb7..33bb39c2 100644 --- a/postgresql/test/test_python.py +++ b/postgresql/test/test_python.py @@ -139,9 +139,8 @@ class anob(object): class test_socket(unittest.TestCase): def testFindAvailable(self): - # the port is randomly generated, so make a few trials before - # determining success. - for i in range(100): + # Host sanity check; this is likely fragile. + for i in range(4): portnum = find_available_port() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: From 5f57f37f02a063241bde513cb76fc245dd036ed4 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 25 Sep 2020 14:09:00 -0700 Subject: [PATCH 051/109] Likewise, update postgresql.installation's subprocess usage(Popen.communicate). --- postgresql/installation.py | 40 ++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/postgresql/installation.py b/postgresql/installation.py index d8ed57a6..c8d0361f 100644 --- a/postgresql/installation.py +++ b/postgresql/installation.py @@ -18,39 +18,37 @@ from . import string as pg_str # Get the output from the given command. -# *args are transformed into "long options", '--' + x -def get_command_output(exe, *args): +# Variable arguments are transformed into "long options", '--' + x +def get_command_output(exe, *args, encoding='utf-8', timeout=8): pa = list(exe) + [ '--' + x.strip() for x in args if x is not None ] p = subprocess.Popen(pa, close_fds = close_fds, stdout = subprocess.PIPE, - stderr = subprocess.PIPE, - stdin = subprocess.PIPE, + stderr = None, + stdin = None, shell = False ) - p.stdin.close() - p.stderr.close() - while True: - try: - rv = p.wait() - break - except OSError as e: - if e.errno != errno.EINTR: - raise - if rv != 0: + + try: + stdout, stderr = p.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + p.kill() + stdout, stderr = p.communicate(timeout=2) + + if p.returncode != 0: return None - with p.stdout, io.TextIOWrapper(p.stdout) as txt: - return txt.read() -def pg_config_dictionary(*pg_config_path): + return stdout.decode(encoding) + +def pg_config_dictionary(*pg_config_path, encoding='utf-8', timeout=8): """ Create a dictionary of the information available in the given pg_config_path. This provides a one-shot solution to fetching information from the pg_config binary. Returns a dictionary object. """ - default_output = get_command_output(pg_config_path) + default_output = get_command_output(pg_config_path, encoding=encoding, timeout=timeout) if default_output is not None: d = {} for x in default_output.splitlines(): @@ -67,7 +65,7 @@ def pg_config_dictionary(*pg_config_path): # Second, all the -- options except version. # Third, --version as it appears to be exclusive in some cases. opt = [] - for l in get_command_output(pg_config_path, 'help').splitlines(): + for l in get_command_output(pg_config_path, 'help', encoding=encoding, timeout=timeout).splitlines(): dash_pos = l.find('--') if dash_pos == -1: continue @@ -79,8 +77,8 @@ def pg_config_dictionary(*pg_config_path): if 'version' in opt: opt.remove('version') - d=dict(zip(opt, get_command_output(pg_config_path, *opt).splitlines())) - d['version'] = get_command_output(pg_config_path, 'version').strip() + d=dict(zip(opt, get_command_output(pg_config_path, *opt, encoding=encoding, timeout=timeout).splitlines())) + d['version'] = get_command_output(pg_config_path, 'version', encoding=encoding, timeout=timeout).strip() return d ## From 6eac536ecf7618105a48e4c77d2ec818f70810e6 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 25 Sep 2020 14:10:48 -0700 Subject: [PATCH 052/109] Using str.decode for command output now. --- postgresql/installation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/postgresql/installation.py b/postgresql/installation.py index c8d0361f..046bff44 100644 --- a/postgresql/installation.py +++ b/postgresql/installation.py @@ -8,7 +8,6 @@ import os import os.path import subprocess -import io import errno from itertools import cycle, chain from operator import itemgetter From 1e14468faf737673f80e55bb589e3817e8076fde Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 1 Oct 2020 21:52:00 -0700 Subject: [PATCH 053/109] Open 1.3. --- postgresql/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/project.py b/postgresql/project.py index f7c4f7bd..c4d8516a 100644 --- a/postgresql/project.py +++ b/postgresql/project.py @@ -8,5 +8,5 @@ meaculpa = 'Python+Postgres' abstract = 'Driver and tools library for PostgreSQL' -version_info = (1, 2, 2) +version_info = (1, 3, 0) version = '.'.join(map(str, version_info)) From 9caff9a49c90eb60498d780bdb3ff920dcc2ef18 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 1 Oct 2020 21:59:59 -0700 Subject: [PATCH 054/109] Apply DB-API 2.0 exception correction. --- postgresql/documentation/changes-v1.3.rst | 7 +++++++ postgresql/driver/dbapi20.py | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 postgresql/documentation/changes-v1.3.rst diff --git a/postgresql/documentation/changes-v1.3.rst b/postgresql/documentation/changes-v1.3.rst new file mode 100644 index 00000000..962887c5 --- /dev/null +++ b/postgresql/documentation/changes-v1.3.rst @@ -0,0 +1,7 @@ +Changes in v1.3 +=============== + +1.3.0 +----- + + * Commit DB-API 2.0 ClientCannotConnect exception correction. diff --git a/postgresql/driver/dbapi20.py b/postgresql/driver/dbapi20.py index d0a2ac6b..dafd49af 100644 --- a/postgresql/driver/dbapi20.py +++ b/postgresql/driver/dbapi20.py @@ -325,7 +325,7 @@ class Connection(Connection): NotSupportedError = NotSupportedError # Explicitly manage DB-API connected state to properly - # throw the already closed error. This will be active in 1.3. + # throw the already closed error. _dbapi_connected_flag = False def autocommit_set(self, val): @@ -361,7 +361,7 @@ def connect(self, *args, **kw): self._dbapi_connected_flag = True def close(self): - if self.closed:# and self._dbapi_connected_flag: + if self.closed and self._dbapi_connected_flag: raise Error( "connection already closed", source = 'CLIENT', From 1467310aa91181c512b72cb1f85f0bdfbdd47d0a Mon Sep 17 00:00:00 2001 From: James William Pye Date: Mon, 26 Oct 2020 09:12:15 -0700 Subject: [PATCH 055/109] Formatting adjustments; remove types-as-docs and avoid single line doc-strings. --- postgresql/clientparameters.py | 18 ++-- postgresql/cluster.py | 50 +++++----- postgresql/configfile.py | 20 ++-- postgresql/copyman.py | 2 +- postgresql/driver/pq3.py | 125 ++++++++++++++----------- postgresql/exceptions.py | 144 +++++++++++++++++++++-------- postgresql/iri.py | 18 +++- postgresql/message.py | 10 +- postgresql/notifyman.py | 6 +- postgresql/pgpassfile.py | 12 ++- postgresql/protocol/client3.py | 20 +++- postgresql/protocol/element3.py | 47 +++++++--- postgresql/protocol/pbuffer.py | 25 ++++- postgresql/protocol/xact3.py | 13 +-- postgresql/release/__init__.py | 2 +- postgresql/release/distutils.py | 8 +- postgresql/resolved/riparse.py | 24 +++-- postgresql/string.py | 22 +++-- postgresql/sys.py | 10 +- postgresql/test/test_cluster.py | 4 +- postgresql/test/test_configfile.py | 2 +- postgresql/types/__init__.py | 6 +- postgresql/versionstring.py | 30 +++--- 23 files changed, 389 insertions(+), 229 deletions(-) diff --git a/postgresql/clientparameters.py b/postgresql/clientparameters.py index 59ed876b..0f69d6fb 100644 --- a/postgresql/clientparameters.py +++ b/postgresql/clientparameters.py @@ -580,15 +580,15 @@ def resolve_pg_service_file( return None def collect( - parsed_options = None, - no_defaults = False, - environ = os.environ, - environ_prefix = 'PG', - default_pg_sysconfdir = None, - pg_service_file = None, - prompt_title = '', - parameters = (), - ): + parsed_options = None, + no_defaults = False, + environ = os.environ, + environ_prefix = 'PG', + default_pg_sysconfdir = None, + pg_service_file = None, + prompt_title = '', + parameters = (), +): """ Build a normalized client parameters dictionary for use with a connection construction interface. diff --git a/postgresql/cluster.py b/postgresql/cluster.py index 2b3f8ac3..bb0ff0d9 100644 --- a/postgresql/cluster.py +++ b/postgresql/cluster.py @@ -40,23 +40,35 @@ class ClusterError(pg_exc.Error): code = '-C000' source = 'CLUSTER' class ClusterInitializationError(ClusterError): - "General cluster initialization failure" + """ + General cluster initialization failure. + """ code = '-Cini' class InitDBError(ClusterInitializationError): - "A non-zero result was returned by the initdb command" + """ + A non-zero result was returned by the initdb command. + """ code = '-Cidb' class ClusterStartupError(ClusterError): - "Cluster startup failed" + """ + Cluster startup failed. + """ code = '-Cbot' class ClusterNotRunningError(ClusterError): - "Cluster is not running" + """ + Cluster is not running. + """ code = '-Cdwn' class ClusterTimeoutError(ClusterError): - "Cluster operation timed out" + """ + Cluster operation timed out. + """ code = '-Cout' class ClusterWarning(pg_exc.Warning): - "Warning issued by cluster operations" + """ + Warning issued by cluster operations. + """ code = '-Cwrn' source = 'CLUSTER' @@ -154,10 +166,7 @@ def hba_file(self, join = os.path.join): join(self.data_directory, self.DEFAULT_HBA_FILENAME) ) - def __init__(self, - installation, - data_directory, - ): + def __init__(self, installation, data_directory): self.installation = installation self.data_directory = os.path.abspath(data_directory) self.pgsql_dot_conf = os.path.join( @@ -190,11 +199,7 @@ def __exit__(self, typ, val, tb): self.stop() self.wait_until_stopped() - def init(self, - password = None, - timeout = None, - **kw - ): + def init(self, password = None, timeout = None, **kw): """ Create the cluster at the given `data_directory` using the provided keyword parameters as options to the command. @@ -323,10 +328,7 @@ def drop(self): os.rmdir(os.path.join(root, name)) os.rmdir(self.data_directory) - def start(self, - logfile = None, - settings = None - ): + def start(self, logfile = None, settings = None): """ Start the cluster. """ @@ -562,10 +564,7 @@ def ready_for_connections(self): # credentials... strange, but true.. return e if e is not None else True - def wait_until_started(self, - timeout = 10, - delay = 0.05, - ): + def wait_until_started(self, timeout = 10, delay = 0.05): """ After the `start` method is used, this can be ran in order to block until the cluster is ready for use. @@ -614,10 +613,7 @@ def wait_until_started(self, raise e time.sleep(delay) - def wait_until_stopped(self, - timeout = 10, - delay = 0.05 - ): + def wait_until_stopped(self, timeout = 10, delay = 0.05): """ After the `stop` method is used, this can be ran in order to block until the cluster is shutdown. diff --git a/postgresql/configfile.py b/postgresql/configfile.py index f20a04b0..18d312e8 100644 --- a/postgresql/configfile.py +++ b/postgresql/configfile.py @@ -1,7 +1,9 @@ ## # .configfile ## -'PostgreSQL configuration file parser and editor functions.' +""" +PostgreSQL configuration file parser and editor functions. +""" import sys import os from . import string as pg_str @@ -76,18 +78,18 @@ def unquote(s, quote = quote): return s[1:-1].replace(quote*2, quote) def write_config(map, writer, keys = None): - 'A configuration writer that will trample & merely write the settings' + """ + A configuration writer that will trample & merely write the settings. + """ if keys is None: keys = map for k in keys: writer('='.join((k, map[k])) + os.linesep) -def alter_config( - map : "the configuration changes to make", - fo : "file object containing configuration lines(Iterable)", - keys : "the keys to change; defaults to map.keys()" = None -): - 'Alters a configuration file without trampling on the existing structure' +def alter_config(map, fo, keys = None): + """ + Alters a configuration file without trampling on the existing structure. + """ if keys is None: keys = list(map.keys()) # Normalize keys and map them back to @@ -212,7 +214,7 @@ class ConfigFile(pg_api.Settings): """ Provides a mapping interface to a configuration file. - Every action will cause the file to be wholly read, so using `update` to make + Every operation will cause the file to be wholly read, so using `update` to make multiple changes is desirable. """ _e_factors = ('path',) diff --git a/postgresql/copyman.py b/postgresql/copyman.py index 70ef7e51..b0f7a6c0 100644 --- a/postgresql/copyman.py +++ b/postgresql/copyman.py @@ -443,7 +443,7 @@ def __next__(self): return self.nextchunk() def __init__(self, - recv_into : "callable taking writable buffer and size", + recv_into, buffer_size = default_buffer_size ): super().__init__() diff --git a/postgresql/driver/pq3.py b/postgresql/driver/pq3.py index b4695af5..a42d0bab 100644 --- a/postgresql/driver/pq3.py +++ b/postgresql/driver/pq3.py @@ -219,17 +219,18 @@ def type_from_oid(self, oid): return typ def resolve_descriptor(self, desc, index): - 'create a sequence of I/O routines from a pq descriptor' + """ + Create a sequence of I/O routines from a pq descriptor. + """ return [ (self.resolve(x[3]) or (None, None))[index] for x in desc ] # lookup a type's IO routines from a given typid def resolve(self, - typid : "The Oid of the type to resolve pack and unpack routines for.", - from_resolution_of : \ - "Sequence of typid's used to identify infinite recursion" = (), - builtins : "types.io.resolve" = pg_types_io.resolve, + typid : int, + from_resolution_of : [int] = (), + builtins = pg_types_io.resolve, quote_ident = quote_ident ): if from_resolution_of and typid in from_resolution_of: @@ -406,17 +407,19 @@ def RowTypeFactory(self, attribute_map = {}, _Row = pg_types.Row.from_sequence, # record_io_factory - Build an I/O pair for RECORDs ## def record_io_factory(self, - column_io : "sequence (pack,unpack) tuples corresponding to the columns", - typids : "sequence of type Oids; index must correspond to the composite's", - attmap : "mapping of column name to index number", - typnames : "sequence of sql type names in order", - attnames : "sequence of attribute names in order", - composite_relid : "oid of the composite relation", - composite_name : "the name of the composite type", + column_io, typids, attmap, typnames, attnames, composite_relid, composite_name, get0 = get0, get1 = get1, fmt_errmsg = "failed to {0} attribute {1}, {2}::{3}, of composite {4} from wire data".format ): + # column_io: sequence (pack,unpack) tuples corresponding to the columns. + # typids: sequence of type Oids; index must correspond to the composite's. + # attmap: mapping of column name to index number. + # typnames: sequence of sql type names in order. + # attnames: sequence of attribute names in order. + # composite_relid: oid of the composite relation. + # composite_name: the name of the composite type. + fpack = tuple(map(get0, column_io)) funpack = tuple(map(get1, column_io)) row_constructor = self.RowTypeFactory(attribute_map = attmap, composite_relid = composite_relid) @@ -448,18 +451,18 @@ def raise_unpack_tuple_error(cause, procs, tup, itemnum): )), cause = cause) def unpack_a_record(data, - unpack = io_lib.record_unpack, - process_tuple = process_tuple, - row_constructor = row_constructor - ): + unpack = io_lib.record_unpack, + process_tuple = process_tuple, + row_constructor = row_constructor + ): data = tuple([x[1] for x in unpack(data)]) return row_constructor(process_tuple(funpack, data, raise_unpack_tuple_error)) sorted_atts = sorted(attmap.items(), key = get1) def pack_a_record(data, - pack = io_lib.record_pack, - process_tuple = process_tuple, - ): + pack = io_lib.record_pack, + process_tuple = process_tuple, + ): if isinstance(data, dict): data = [data.get(k) for k,_ in sorted_atts] return pack( @@ -765,8 +768,8 @@ def _process_tuple_chunk_Column(self, x, range = range): # Process the element.Tuple message in x for rows() def _process_tuple_chunk_Row(self, x, - proc = process_chunk, - ): + proc = process_chunk, + ): rc = self._row_constructor return [ rc(y) @@ -778,7 +781,7 @@ def _process_tuple_chunk(self, x, proc = process_chunk): return proc(self._output_io, x, self._raise_column_tuple_error) def _raise_column_tuple_error(self, cause, procs, tup, itemnum): - 'for column processing' + # For column processing. # The element traceback will include the full list of parameters. data = repr(tup[itemnum]) if len(data) > 80: @@ -835,12 +838,16 @@ def sql_column_types(self): ] def command(self): - "The completion message's command identifier" + """ + The completion message's command identifier. + """ if self._complete_message is not None: return self._complete_message.extract_command().decode('ascii') def count(self): - "The completion message's count number" + """ + The completion message's count number. + """ if self._complete_message is not None: return self._complete_message.extract_count() @@ -2198,7 +2205,7 @@ def start(self): @staticmethod def _release_string(id): - 'release "";' + # Release ""; return 'RELEASE "xact(' + id.replace('"', '""') + ')";' def commit(self): @@ -2311,13 +2318,13 @@ def execute(self, query : str) -> None: self._pq_complete() def do(self, language : str, source : str, - qlit = pg_str.quote_literal, - qid = pg_str.quote_ident, - ) -> None: + qlit = pg_str.quote_literal, + qid = pg_str.quote_ident, + ) -> None: sql = "DO " + qlit(source) + " LANGUAGE " + qid(language) + ";" self.execute(sql) - def xact(self, isolation = None, mode = None): + def xact(self, isolation = None, mode = None) -> Transaction: x = Transaction(self, isolation = isolation, mode = mode) return x @@ -2328,6 +2335,8 @@ def prepare(self, ) -> Statement: ps = Class(self, statement_id, sql_statement_string) ps._init() + + # Complete protocol transaction to maintain point of origin in error cases. ps._fini() return ps @@ -2412,7 +2421,9 @@ def __enter__(self): return self def connect(self): - 'Establish the connection to the server' + """ + Establish the connection to the server. + """ if self.closed is False: # already connected? just return. return @@ -2623,11 +2634,11 @@ def _pq_step(self, complete_state = globals()['xact'].Complete): del self._controller def _receive_async(self, - msg, controller = None, - showoption = element.ShowOption.type, - notice = element.Notice.type, - notify = element.Notify.type, - ): + msg, controller = None, + showoption = element.ShowOption.type, + notice = element.Notice.type, + notify = element.Notify.type, + ): c = controller or getattr(self, '_controller', self) typ = msg.type if typ == showoption: @@ -2767,12 +2778,12 @@ def socket_factory_sequence(self): def __init__(self, connect_timeout : int = None, - server_encoding : "server encoding hint for driver" = None, + server_encoding = None, sslmode : ('allow', 'prefer', 'require', 'disable') = None, - sslcrtfile : "filepath" = None, - sslkeyfile : "filepath" = None, - sslrootcrtfile : "filepath" = None, - sslrootcrlfile : "filepath" = None, + sslcrtfile = None, + sslkeyfile = None, + sslrootcrtfile = None, + sslrootcrlfile = None, driver = None, **kw ): @@ -2837,7 +2848,9 @@ def __init__(self, # class Connector class SocketConnector(Connector): - 'abstract connector for using `socket` and `ssl`' + """ + Abstract connector for using `socket` and `ssl`. + """ @abstractmethod def socket_factory_sequence(self): """ @@ -2872,12 +2885,14 @@ def __init__(self, host, port, ipv, **kw): super().__init__(**kw) class IP4(IPConnector): - 'Connector for establishing IPv4 connections' + """ + Connector for establishing IPv4 connections. + """ ipv = 4 address_family = socket.AF_INET def __init__(self, - host : "IPv4 Address (str)" = None, + host : str = None, port : int = None, ipv = 4, **kw @@ -2885,12 +2900,14 @@ def __init__(self, super().__init__(host, port, ipv, **kw) class IP6(IPConnector): - 'Connector for establishing IPv6 connections' + """ + Connector for establishing IPv6 connections. + """ ipv = 6 address_family = socket.AF_INET6 def __init__(self, - host : "IPv6 Address (str)" = None, + host : str = None, port : int = None, ipv = 6, **kw @@ -2898,7 +2915,9 @@ def __init__(self, super().__init__(host, port, ipv, **kw) class Unix(SocketConnector): - 'Connector for establishing unix domain socket connections' + """ + Connector for establishing unix domain socket connections. + """ def socket_factory_sequence(self): return self._socketcreators @@ -2946,7 +2965,7 @@ def __init__(self, host : str = None, port : (str, int) = None, ipv : int = None, - address_family : "address family to use(AF_INET,AF_INET6)" = None, + address_family = None, **kw ): if host is None: @@ -2992,10 +3011,7 @@ def fit(self, **kw ) -> Connector: """ - Create the appropriate `postgresql.api.Connector` based on the - parameters. - - This also protects against mutually exclusive parameters. + Create the appropriate `postgresql.api.Connector` based on the parameters. """ if unix is not None: if host is not None: @@ -3006,7 +3022,7 @@ def fit(self, else: if host is None or port is None: raise TypeError("'host' and 'port', or 'unix' must be supplied") - # We have a host and a port. + # If it's an IP address, IP4 or IP6 should be selected. if ':' in host: # There's a ':' in host, good chance that it's IPv6. @@ -3016,7 +3032,7 @@ def fit(self, except (socket.error, NameError): pass - # Not IPv6, maybe IPv4... + # Not IPv6, maybe IPv4. try: socket.inet_aton(host) # It's IP4 @@ -3029,6 +3045,9 @@ def fit(self, def connect(self, **kw) -> Connection: """ + Create an established Connection instance from a temporary Connector + built using the given keywords. + For information on acceptable keywords, see: `postgresql.documentation.driver`:Connection Keywords diff --git a/postgresql/exceptions.py b/postgresql/exceptions.py index fc960794..39cf5a33 100644 --- a/postgresql/exceptions.py +++ b/postgresql/exceptions.py @@ -36,14 +36,20 @@ PythonException = Exception class Exception(Exception): - 'Base PostgreSQL exception class' + """ + Base PostgreSQL exception class. + """ pass class LoadError(Exception): - 'Failed to load a library' + """ + Failed to load a library. + """ class Disconnection(Exception): - 'Exception identifying errors that result in disconnection' + """ + Exception identifying errors that result in disconnection. + """ class Warning(Message): code = '01000' @@ -80,12 +86,16 @@ class NoMoreSetsReturned(NoDataWarning): code = '02001' class Error(Message, Exception): - 'A PostgreSQL Error' + """ + A PostgreSQL Error. + """ _e_label = 'ERROR' code = '' def __str__(self): - 'Call .sys.errformat(self)' + """ + Call .sys.errformat(self). + """ return pg_sys.errformat(self) @property @@ -94,7 +104,9 @@ def fatal(self): return None if f is None else f in ('PANIC', 'FATAL') class DriverError(Error): - "Errors originating in the driver's implementation." + """ + Errors originating in the driver's implementation. + """ source = 'CLIENT' code = '--000' class AuthenticationMethodError(DriverError, Disconnection): @@ -109,7 +121,9 @@ class InsecurityError(DriverError, Disconnection): """ code = '--SEC' class ConnectTimeoutError(DriverError, Disconnection): - 'Client was unable to esablish a connection in the given time' + """ + Client was unable to esablish a connection in the given time. + """ code = '--TOE' class TypeIOError(DriverError): @@ -144,7 +158,9 @@ class ConnectionDoesNotExistError(ConnectionError): """ code = '08003' class ConnectionFailureError(ConnectionError): - 'Raised when a connection is dropped' + """ + Raised when a connection is dropped. + """ code = '08006' class ClientCannotConnectError(ConnectionError): @@ -164,7 +180,9 @@ class TriggeredActionError(Error): code = '09000' class FeatureError(Error): - "Unsupported feature" + """ + "Unsupported feature. + """ code = '0A000' class TransactionInitiationError(TransactionError): @@ -187,7 +205,9 @@ class CaseNotFoundError(Error): code = '20000' class CardinalityError(Error): - "Wrong number of rows returned" + """ + Wrong number of rows returned. + """ code = '21000' class TriggeredDataChangeViolation(Error): @@ -197,13 +217,17 @@ class AuthenticationSpecificationError(Error, Disconnection): code = '28000' class DPDSEError(Error): - "Dependent Privilege Descriptors Still Exist" + """ + Dependent Privilege Descriptors Still Exist. + """ code = '2B000' class DPDSEObjectError(DPDSEError): code = '2BP01' class SREError(Error): - "SQL Routine Exception" + """ + SQL Routine Exception. + """ code = '2F000' class FunctionExecutedNoReturnStatementError(SREError): code = '2F005' @@ -215,7 +239,9 @@ class ReadingDataProhibitedError(SREError): code = '2F004' class EREError(Error): - "External Routine Exception" + """ + External Routine Exception. + """ code = '38000' class ContainingSQLNotPermittedError(EREError): code = '38001' @@ -227,7 +253,9 @@ class ReadingSQLDataNotPermittedError(EREError): code = '38004' class ERIEError(Error): - "External Routine Invocation Exception" + """ + External Routine Invocation Exception. + """ code = '39000' class InvalidSQLState(ERIEError): code = '39001' @@ -239,7 +267,9 @@ class SRFProtocolError(ERIEError): code = '39P02' class TRError(TransactionError): - "Transaction Rollback" + """ + Transaction Rollback. + """ code = '40000' class DeadlockError(TRError): code = '40P01' @@ -252,7 +282,9 @@ class StatementCompletionUnknownError(TRError): class ITSError(TransactionError): - "Invalid Transaction State" + """ + Invalid Transaction State. + """ code = '25000' class ActiveTransactionError(ITSError): code = '25001' @@ -265,24 +297,34 @@ class BadIsolationForBranchError(ITSError): class NoActiveTransactionForBranchError(ITSError): code = '25005' class ReadOnlyTransactionError(ITSError): - "Occurs when an alteration occurs in a read-only transaction." + """ + Occurs when an alteration occurs in a read-only transaction. + """ code = '25006' class SchemaAndDataStatementsError(ITSError): - "Mixed schema and data statements not allowed." + """ + Mixed schema and data statements not allowed. + """ code = '25007' class InconsistentCursorIsolationError(ITSError): - "The held cursor requires the same isolation." + """ + The held cursor requires the same isolation. + """ code = '25008' class NoActiveTransactionError(ITSError): code = '25P01' class InFailedTransactionError(ITSError): - "Occurs when an action occurs in a failed transaction." + """ + Occurs when an action occurs in a failed transaction. + """ code = '25P02' class SavepointError(TransactionError): - "Classification error designating errors that relate to savepoints." + """ + Classification error designating errors that relate to savepoints. + """ code = '3B000' class InvalidSavepointSpecificationError(SavepointError): code = '3B001' @@ -291,7 +333,9 @@ class TransactionTerminationError(TransactionError): code = '2D000' class IRError(Error): - "Insufficient Resource Error" + """ + Insufficient Resource Error. + """ code = '53000' class MemoryError(IRError, MemoryError): code = '53200' @@ -301,7 +345,9 @@ class TooManyConnectionsError(IRError): code = '53300' class PLEError(OverflowError): - "Program Limit Exceeded" + """ + Program Limit Exceeded + """ code = '54000' class ComplexityOverflowError(PLEError): code = '54001' @@ -311,7 +357,9 @@ class ArgumentOverflowError(PLEError): code = '54023' class ONIPSError(Error): - "Object Not In Prerequisite State" + """ + Object Not In Prerequisite State. + """ code = '55000' class ObjectInUseError(ONIPSError): code = '55006' @@ -322,7 +370,9 @@ class UnavailableLockError(ONIPSError): class SEARVError(Error): - "Syntax Error or Access Rule Violation" + """ + Syntax Error or Access Rule Violation. + """ code = '42000' class SEARVNameError(SEARVError): @@ -445,7 +495,9 @@ class SchemaNameError(NameError): code = '3F000' class ICVError(Error): - "Integrity Contraint Violation" + """ + Integrity Contraint Violation. + """ code = '23000' class RestrictError(ICVError): code = '23001' @@ -539,7 +591,9 @@ class EscapeSequenceError(DataError): class EscapeCharacterConflictError(DataError): code = '2200B' class EscapeCharacterError(DataError): - "Invalid escape character" + """ + Invalid escape character. + """ code = '2200C' class SubstringError(DataError): @@ -573,7 +627,9 @@ class IndexCorruptedError(InternalError): code = 'XX002' class SIOError(Error): - "System I/O" + """ + System I/O. + """ code = '58000' class UndefinedFileError(SIOError): code = '58P01' @@ -581,13 +637,17 @@ class DuplicateFileError(SIOError): code = '58P02' class CFError(Error): - "Configuration File Error" + """ + Configuration File Error. + """ code = 'F0000' class LockFileExistsError(CFError): code = 'F0001' class OIError(Error): - "Operator Intervention" + """ + Operator Intervention. + """ code = '57000' class QueryCanceledError(OIError): code = '57014' @@ -596,14 +656,20 @@ class AdminShutdownError(OIError, Disconnection): class CrashShutdownError(OIError, Disconnection): code = '57P02' class ServerNotReadyError(OIError, Disconnection): - 'Thrown when a connection is established to a server that is still starting up.' + """ + Thrown when a connection is established to a server that is still starting up. + """ code = '57P03' class PLPGSQLError(Error): - "Error raised by a PL/PgSQL procedural function" + """ + Error raised by a PL/PgSQL procedural function. + """ code = 'P0000' class PLPGSQLRaiseError(PLPGSQLError): - "Error raised by a PL/PgSQL RAISE statement." + """ + Error raised by a PL/PgSQL RAISE statement. + """ code = 'P0001' class PLPGSQLNoDataFoundError(PLPGSQLError): code = 'P0002' @@ -615,9 +681,9 @@ class PLPGSQLTooManyRowsError(PLPGSQLError): code_to_error = {} code_to_warning = {} def map_errors_and_warnings( - objs : "A iterable of `Warning`s and `Error`'s", - error_container : "apply the code to error association to this object" = code_to_error, - warning_container : "apply the code to warning association to this object" = code_to_warning, + objs, + error_container = code_to_error, + warning_container = code_to_warning, ): """ Construct the code-to-error and code-to-warning associations. @@ -655,9 +721,9 @@ def map_errors_and_warnings( container[obj.pg_code] = obj def code_lookup( - default : "The object to return when no code or class is found", - container : "where to look for the object associated with the code", - code : "the code to find the exception for" + default, + container, + code ): obj = container.get(code) if obj is None: diff --git a/postgresql/iri.py b/postgresql/iri.py index 5e635dfa..1ace1ba8 100644 --- a/postgresql/iri.py +++ b/postgresql/iri.py @@ -6,7 +6,7 @@ PQ IRIs take the form:: - pq://user:pass@host:port/database?setting=value&setting2=value2#public,othernamespace + pq://user:pass@host:port/database?setting=value&setting2=value2 IPv6 is supported via the standard representation:: @@ -27,7 +27,9 @@ escape_path_re = re.compile('[%s]' %(re.escape(ri.unescaped + ','),)) def structure(d, fieldproc = ri.unescape): - 'Create a clientparams dictionary from a parsed RI' + """ + Create a clientparams dictionary from a parsed RI. + """ if d.get('scheme', 'pq').lower() != 'pq': raise ValueError("PQ-IRI scheme is not 'pq'") cpd = { @@ -90,7 +92,9 @@ def construct_path(x, re = escape_path_re): return ','.join((re.sub(ri.re_pct_encode, y) for y in x)) def construct(x, obscure_password = False): - 'Construct a RI dictionary from a clientparams dictionary' + """ + Construct a RI dictionary from a clientparams dictionary. + """ # the rather exhaustive settings choreography is due to # a desire to allow the search_path to be appended in the fragment settings = x.get('settings') @@ -167,7 +171,9 @@ def construct(x, obscure_password = False): ) def parse(s, fieldproc = ri.unescape): - 'Parse a Postgres IRI into a dictionary object' + """ + Parse a Postgres IRI into a dictionary object. + """ return structure( # In ri.parse, don't unescape the parsed values as our sub-structure # uses the escape mechanism in IRIs to specify literal separator @@ -177,7 +183,9 @@ def parse(s, fieldproc = ri.unescape): ) def serialize(x, obscure_password = False): - 'Return a Postgres IRI from a dictionary object.' + """ + Return a Postgres IRI from a dictionary object. + """ return ri.unsplit(construct(x, obscure_password = obscure_password)) if __name__ == '__main__': diff --git a/postgresql/message.py b/postgresql/message.py index e6000c02..7bbe77d9 100644 --- a/postgresql/message.py +++ b/postgresql/message.py @@ -64,11 +64,11 @@ def isconsistent(self, other): ) def __init__(self, - message : "The primary information of the message", - code : "Message code to attach (SQL state)" = None, - details : "additional information associated with the message" = {}, - source : "Which side generated the message(SERVER, CLIENT)" = None, - creator : "The interface element that called for instantiation" = None, + message, + code = None, + details = {}, + source = None, + creator = None, ): self.message = message self.details = details diff --git a/postgresql/notifyman.py b/postgresql/notifyman.py index b46aa0df..cc6ef2c8 100644 --- a/postgresql/notifyman.py +++ b/postgresql/notifyman.py @@ -106,7 +106,7 @@ def trash(self, connections): def queue(self, db, notifies): """ - Queue the notifies for the specified connection. Upon success, the + Queue the notifies for the specified connection. This method can be overridden by subclasses to take a callback approach to notification management. @@ -186,7 +186,9 @@ def settimeout(self, seconds): self.timeout = seconds def gettimeout(self): - 'Get the timeout.' + """ + Get the timeout assigned by `settimeout`. + """ return self.timeout def __iter__(self): diff --git a/postgresql/pgpassfile.py b/postgresql/pgpassfile.py index e7a505a7..ee0ae73f 100644 --- a/postgresql/pgpassfile.py +++ b/postgresql/pgpassfile.py @@ -1,7 +1,9 @@ ## # .pgpassfile - parse and lookup passwords in a pgpassfile ## -'Parse pgpass files and subsequently lookup a password.' +""" +Parse pgpass files and subsequently lookup a password. +""" import os.path def split(line, len = len): @@ -30,7 +32,9 @@ def split(line, len = len): return r def parse(data): - 'produce a list of [(word, (host,port,dbname,user))] from a pgpass file object' + """ + Produce a list of [(word, (host,port,dbname,user))] from a pgpass file object. + """ return [ (x[-1], x[0:4]) for x in [split(line) for line in data] if x ] @@ -50,7 +54,9 @@ def lookup_password(words, uhpd): return word def lookup_password_file(path, t): - 'like lookup_password, but takes a file path' + """ + Like lookup_password, but takes a file path. + """ with open(path) as f: return lookup_password(parse(f), t) diff --git a/postgresql/protocol/client3.py b/postgresql/protocol/client3.py index daba076f..f7e21750 100644 --- a/postgresql/protocol/client3.py +++ b/postgresql/protocol/client3.py @@ -269,7 +269,9 @@ def read_into(self, Complete = xact.Complete): return True def standard_read_messages(self): - 'read more messages into self.read when self.read is empty' + """ + Read more messages into self.read when self.read is empty. + """ r = True if not self.read: # get more data from the wire and @@ -314,7 +316,9 @@ def send_message_data(self): def standard_write_messages(self, messages, cat_messages = element.cat_messages ): - 'protocol message writer' + """ + Protocol message writer. + """ if self.writing is not self.written: self.message_data += cat_messages(self.writing) self.written = self.writing @@ -327,7 +331,9 @@ def standard_write_messages(self, messages, write_messages = standard_write_messages def traced_write_messages(self, messages): - 'message_writer used when tracing' + """ + `message_writer` used when tracing. + """ for msg in messages: t = getattr(msg, 'type', None) if t is not None: @@ -346,7 +352,9 @@ def traced_write_messages(self, messages): return self.standard_write_messages(messages) def traced_read_messages(self): - 'message_reader used when tracing' + """ + `message_reader` used when tracing. + """ r = self.standard_read_messages() for msg in self.read: self._tracer('↓ %r(%d): %r%s' %( @@ -433,7 +441,9 @@ def step(self): self.xact = None def complete(self): - 'complete the current transaction' + """ + Complete the current transaction. + """ # Continue to transition until all transactions have been # completed, or an exception occurs that does not signal retry. x = self.xact diff --git a/postgresql/protocol/element3.py b/postgresql/protocol/element3.py index 39fca329..e5a95781 100644 --- a/postgresql/protocol/element3.py +++ b/postgresql/protocol/element3.py @@ -135,8 +135,7 @@ def __repr__(self): class Void(Message): """ - An absolutely empty message. When serialized, it always yields an empty - string. + An absolutely empty message. When serialized, it always yields an empty string. """ type = b'' __slots__ = () @@ -146,7 +145,7 @@ def bytes(self): def serialize(self): return b'' - + def __new__(typ, *args, **kw): return VoidMessage VoidMessage = Message.__new__(Void) @@ -178,7 +177,9 @@ def parse(typ, data): return typ((data[0:1], data[5:])) class EmptyMessage(Message): - 'An abstract message that is always empty' + """ + An abstract message that is always empty. + """ __slots__ = () type = b'' @@ -195,7 +196,9 @@ def parse(typ, data): return typ.SingleInstance class Notify(Message): - 'Asynchronous notification message' + """ + Asynchronous notification message. + """ type = message_types[b'A'[0]] __slots__ = ('pid', 'channel', 'payload',) @@ -216,8 +219,9 @@ def parse(typ, data): return typ(pid, channel, payload) class ShowOption(Message): - """ShowOption(name, value) - GUC variable information from backend""" + """ + GUC variable information from backend + """ type = message_types[b'S'[0]] __slots__ = ('name', 'value') @@ -233,7 +237,9 @@ def parse(typ, data): return typ(*(data.split(b'\x00', 2)[0:2])) class Complete(StringMessage): - 'Command completion message.' + """ + Command completion message. + """ type = message_types[b'C'[0]] __slots__ = () @@ -260,42 +266,54 @@ def extract_command(self): return self.data.strip(b'\c\n\t 0123456789') or None class Null(EmptyMessage): - 'Null command' + """ + Null command. + """ type = message_types[b'I'[0]] __slots__ = () NullMessage = Message.__new__(Null) Null.SingleInstance = NullMessage class NoData(EmptyMessage): - 'Null command' + """ + Null command. + """ type = message_types[b'n'[0]] __slots__ = () NoDataMessage = Message.__new__(NoData) NoData.SingleInstance = NoDataMessage class ParseComplete(EmptyMessage): - 'Parse reaction' + """ + Parse reaction. + """ type = message_types[b'1'[0]] __slots__ = () ParseCompleteMessage = Message.__new__(ParseComplete) ParseComplete.SingleInstance = ParseCompleteMessage class BindComplete(EmptyMessage): - 'Bind reaction' + """ + Bind reaction. + """ type = message_types[b'2'[0]] __slots__ = () BindCompleteMessage = Message.__new__(BindComplete) BindComplete.SingleInstance = BindCompleteMessage class CloseComplete(EmptyMessage): - 'Close statement or Portal' + """ + Close statement or Portal. + """ type = message_types[b'3'[0]] __slots__ = () CloseCompleteMessage = Message.__new__(CloseComplete) CloseComplete.SingleInstance = CloseCompleteMessage class Suspension(EmptyMessage): - 'Portal was suspended, more tuples for reading' + """ + Portal was suspended, more tuples for reading. + """ type = message_types[b's'[0]] __slots__ = () SuspensionMessage = Message.__new__(Suspension) @@ -859,7 +877,6 @@ class Function(Message): """ Execute the specified function with the given arguments """ - type = message_types[b'F'[0]] __slots__ = ('oid', 'aformats', 'arguments', 'rformat') diff --git a/postgresql/protocol/pbuffer.py b/postgresql/protocol/pbuffer.py index a3014942..d41a79e0 100644 --- a/postgresql/protocol/pbuffer.py +++ b/postgresql/protocol/pbuffer.py @@ -16,7 +16,10 @@ xl_unpack = struct.Struct('!xL').unpack_from class pq_message_stream(object): - 'provide a message stream from a data stream' + """ + Provide a message stream from a data stream. + """ + _block = 512 _limit = _block * 4 def __init__(self): @@ -24,12 +27,18 @@ def __init__(self): self._start = 0 def truncate(self): - "remove all data in the buffer" + """ + Remove all data in the buffer. + """ + self._strio.truncate(0) self._start = 0 def _rtruncate(self, amt = None): - "[internal] remove the given amount of data" + """ + [internal] remove the given amount of data. + """ + strio = self._strio if amt is None: amt = self._strio.tell() @@ -58,7 +67,10 @@ def _rtruncate(self, amt = None): strio.truncate(size - amt) def has_message(self, xl_unpack = xl_unpack, len = len): - "if the buffer has a message available" + """ + Whether the buffer has a message available. + """ + strio = self._strio strio.seek(self._start) header = strio.read(5) @@ -71,7 +83,10 @@ def has_message(self, xl_unpack = xl_unpack, len = len): return (strio.tell() - self._start) >= length + 1 def __len__(self, xl_unpack = xl_unpack, len = len): - "number of messages in buffer" + """ + Number of messages in buffer. + """ + count = 0 rpos = self._start strio = self._strio diff --git a/postgresql/protocol/xact3.py b/postgresql/protocol/xact3.py index d8e35650..6f5497f6 100644 --- a/postgresql/protocol/xact3.py +++ b/postgresql/protocol/xact3.py @@ -1,7 +1,9 @@ ## # .protocol.xact3 - protocol state machine ## -'PQ version 3.0 client transactions' +""" +PQ version 3.0 client transactions. +""" import sys import os import pprint @@ -94,10 +96,7 @@ class Negotiation(Transaction): """ state = None - def __init__(self, - startup_message : "startup message to send", - password : "password source data(encoded password bytes)", - ): + def __init__(self, startup_message, password): self.startup_message = startup_message self.password = password self.received = [()] @@ -435,7 +434,9 @@ def __repr__(self, format = '{mod}.{name}({nl}{args})'.format): ) def messages_received(self): - 'Received and validate messages' + """ + Received and validate messages. + """ return chain.from_iterable(map(get1, self.completed)) def reverse(self, diff --git a/postgresql/release/__init__.py b/postgresql/release/__init__.py index 36ffdd84..71af8093 100644 --- a/postgresql/release/__init__.py +++ b/postgresql/release/__init__.py @@ -2,5 +2,5 @@ # .release ## """ -Release management code and project meta-data. +Release management code and project/release meta-data. """ diff --git a/postgresql/release/distutils.py b/postgresql/release/distutils.py index e921b7b1..9e018a85 100644 --- a/postgresql/release/distutils.py +++ b/postgresql/release/distutils.py @@ -117,8 +117,8 @@ default_prefix = ['postgresql'] def prefixed_extensions( - prefix : "prefix to prepend to paths" = default_prefix, - extensions_data : "`extensions_data`" = extensions_data, + prefix = default_prefix, + extensions_data = extensions_data, ) -> [Extension]: """ Generator producing the `distutils` `Extension` objects. @@ -134,7 +134,7 @@ def prefixed_extensions( ) def prefixed_packages( - prefix : "prefix to prepend to source paths" = default_prefix, + prefix = default_prefix, packages = subpackages, ): """ @@ -147,7 +147,7 @@ def prefixed_packages( yield prefix + pkg def prefixed_package_data( - prefix : "prefix to prepend to dictionary keys paths" = default_prefix, + prefix = default_prefix, package_data = subpackage_data, ): """ diff --git a/postgresql/resolved/riparse.py b/postgresql/resolved/riparse.py index 668c7119..f91a2618 100644 --- a/postgresql/resolved/riparse.py +++ b/postgresql/resolved/riparse.py @@ -1,7 +1,3 @@ -# -*- encoding: utf-8 -*- -## -# copyright 2008, James William Pye. http://jwp.name -## """ Split, unsplit, parse, serialize, construct and structure resource indicators. @@ -69,7 +65,9 @@ del x def unescape(x, mkval = chr): - 'Substitute percent escapes with literal characters' + """ + Substitute percent escapes with literal characters. + """ nstr = type(x)('') if isinstance(x, str): mkval = chr @@ -193,7 +191,9 @@ def split_path(p, fieldproc = unescape): return [fieldproc(x) for x in p.split('/')] def unsplit(t): - 'Make a RI from a split RI(5-tuple)' + """ + Make a RI from a split RI(5-tuple). + """ s = '' if t[0] is not None: s += t[0] @@ -265,7 +265,9 @@ def split_netloc(netloc, fieldproc = unescape): return (user, password, addr, port) def unsplit_netloc(t): - 'Create a netloc fragment from the given tuple(user,password,host,port)' + """ + Create a netloc fragment from the given tuple(user,password,host,port). + """ if t[0] is None and t[2] is None: return None s = '' @@ -340,7 +342,9 @@ def construct_query(x, ]) def construct(x): - 'Construct a RI tuple(5-tuple) from a dictionary object' + """ + Construct a RI tuple(5-tuple) from a dictionary object. + """ p = x.get('path') if p is not None: p = '/'.join([escape_path_re.sub(re_pct_encode, y) for y in p]) @@ -378,7 +382,9 @@ def parse(s, fieldproc = unescape): return structure(split(s), fieldproc = fieldproc) def serialize(x): - 'Return an RI from a dictionary object. Synonym for ``unsplit(construct(x))``' + """ + Return an RI from a dictionary object. Synonym for ``unsplit(construct(x))``. + """ return unsplit(construct(x)) __docformat__ = 'reStructuredText' diff --git a/postgresql/string.py b/postgresql/string.py index adbd77c4..53799d37 100644 --- a/postgresql/string.py +++ b/postgresql/string.py @@ -16,22 +16,30 @@ import re def escape_literal(text): - "Replace every instance of ' with ''" + """ + Replace every instance of ' with ''. + """ return text.replace("'", "''") def quote_literal(text): - "Escape the literal and wrap it in [single] quotations" + """ + Escape the literal and wrap it in [single] quotations. + """ return "'" + text.replace("'", "''") + "'" def escape_ident(text): - 'Replace every instance of " with ""' + """ + Replace every instance of " with "". + """ return text.replace('"', '""') def needs_quoting(text): return not (text and not text[0].isdecimal() and text.replace('_', 'a').isalnum()) def quote_ident(text): - "Replace every instance of '"' with '""' *and* place '"' on each end" + """ + Replace every instance of '"' with '""' *and* place '"' on each end. + """ return '"' + text.replace('"', '""') + '"' def quote_ident_if_needed(text): @@ -52,7 +60,7 @@ def split(text): """ split the string up by into non-quoted and quoted portions. Zero and even numbered indexes are unquoted portions, while odd indexes are quoted - portions. + portions. Unquoted portions are regular strings, whereas quoted portions are pair-tuples specifying the quotation mechanism and the content thereof. @@ -214,7 +222,9 @@ def split_qname(text, maxsplit = -1): return split_ident(text, maxsplit = maxsplit, sep = '.') def qname(*args): - "Quote the identifiers and join them using '.'" + """ + Quote the identifiers and join them using '.'. + """ return '.'.join([quote_ident(x) for x in args]) def qname_if_needed(*args): diff --git a/postgresql/sys.py b/postgresql/sys.py index 80abe1d6..a8471317 100644 --- a/postgresql/sys.py +++ b/postgresql/sys.py @@ -30,7 +30,7 @@ def default_errformat(val): """ - Built-in error formatter. DON'T TOUCH! + Built-in error formatter. Do not change. """ it = val._e_metas() if val.creator is not None: @@ -85,11 +85,15 @@ def msghook(*args, **kw): return default_msghook(*args, **kw) def reset_errformat(with_func = errformat): - 'restore the original excformat function' + """ + Restore the original excformat function. + """ global errformat errformat = with_func def reset_msghook(with_func = msghook): - 'restore the original msghook function' + """ + Restore the original msghook function. + """ global msghook msghook = with_func diff --git a/postgresql/test/test_cluster.py b/postgresql/test/test_cluster.py index 4f781f6c..b4aee97a 100644 --- a/postgresql/test/test_cluster.py +++ b/postgresql/test/test_cluster.py @@ -82,7 +82,9 @@ def testSuperPassword(self): self.assertEqual(c.prepare('select 1').first(), 1) def testNoParameters(self): - 'simple init and drop' + """ + Simple init and drop. + """ self.init() self.start_cluster() diff --git a/postgresql/test/test_configfile.py b/postgresql/test/test_configfile.py index f57a3c0f..85d30b66 100644 --- a/postgresql/test/test_configfile.py +++ b/postgresql/test/test_configfile.py @@ -237,7 +237,7 @@ def testAroma(self): self.assertTrue( nlines[:4] == lines[:4] ) - + def testSelection(self): # Sanity red = configfile.read_config(['foo = bar'+os.linesep, 'bar = foo']) diff --git a/postgresql/types/__init__.py b/postgresql/types/__init__.py index 6481d929..690e19f3 100644 --- a/postgresql/types/__init__.py +++ b/postgresql/types/__init__.py @@ -286,9 +286,9 @@ def detect_dimensions(hier, len = len): @classmethod def from_elements(typ, - elements : "iterable of elements in the array", - lowerbounds : "beginning of each axis" = None, - upperbounds : "upper bounds; size of each axis" = None, + elements, + lowerbounds = None, + upperbounds = None, len = len, ): """ diff --git a/postgresql/versionstring.py b/postgresql/versionstring.py index ccb39536..04c065a5 100644 --- a/postgresql/versionstring.py +++ b/postgresql/versionstring.py @@ -2,18 +2,16 @@ # .versionstring ## """ -PostgreSQL version parsing. +PostgreSQL version string parsing. ->>> postgresql.version.split('8.0.1') +>>> postgresql.versionstring.split('8.0.1') (8, 0, 1, None, None) """ -def split(vstr : str) -> ( - 'major','minor','patch',...,'state_class','state_level' -): +def split(vstr: str) -> tuple: """ - Split a PostgreSQL version string into a tuple - (major,minor,patch,...,state_class,state_level) + Split a PostgreSQL version string into a tuple. + (major, minor, patch, ..., state_class, state_level) """ v = vstr.strip().split('.') @@ -38,24 +36,22 @@ def split(vstr : str) -> ( vlist += [None] * ((3 - len(vlist)) + 2) return tuple(vlist) -def unsplit(vtup : tuple) -> str: - 'join a version tuple back into the original version string' +def unsplit(vtup: tuple) -> str: + """ + Join a version tuple back into the original version string. + """ svtup = [str(x) for x in vtup[:-2] if x is not None] state_class, state_level = vtup[-2:] - return '.'.join(svtup) + ( - '' if state_class is None else state_class + str(state_level) - ) + return '.'.join(svtup) + ('' if state_class is None else state_class + str(state_level)) -def normalize(split_version : "a tuple returned by `split`") -> tuple: +def normalize(split_version: tuple) -> tuple: """ Given a tuple produced by `split`, normalize the `None` objects into int(0) - or 'final' if it's the ``state_class`` + or 'final' if it's the ``state_class``. """ (*head, state_class, state_level) = split_version mmp = [x if x is not None else 0 for x in head] - return tuple( - mmp + [state_class or 'final', state_level or 0] - ) + return tuple(mmp + [state_class or 'final', state_level or 0]) default_state_class_priority = [ 'dev', From 6b797b7eba00714ef2d2f9cc3b02fa51ba7d0cab Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sat, 5 Dec 2020 19:28:24 -0700 Subject: [PATCH 056/109] Remove query libraries from index and reorder the chapters. --- postgresql/documentation/index.rst | 7 +++---- readthedocs.yml | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/postgresql/documentation/index.rst b/postgresql/documentation/index.rst index 322438d4..715b8c08 100644 --- a/postgresql/documentation/index.rst +++ b/postgresql/documentation/index.rst @@ -14,12 +14,11 @@ Contents admin driver - copyman + clientparameters + cluster notifyman alock - cluster - lib - clientparameters + copyman gotchas Reference diff --git a/readthedocs.yml b/readthedocs.yml index d53ffa68..d75e54ac 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -2,4 +2,4 @@ build: image: latest python: - version: 3.6 + version: 3.7 From 066c0d484f6804d6d47c8cfaba9ba00fc9a7e37e Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sun, 6 Dec 2020 12:42:33 -0700 Subject: [PATCH 057/109] Use triple-quote form docstrings. --- postgresql/python/element.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/postgresql/python/element.py b/postgresql/python/element.py index aa2dc5aa..4257a44e 100644 --- a/postgresql/python/element.py +++ b/postgresql/python/element.py @@ -7,7 +7,9 @@ from .decorlib import propertydoc class RecursiveFactor(Exception): - 'Raised when a factor is ultimately composed of itself' + """ + Raised when a factor is ultimately composed of itself. + """ pass class Element(object, metaclass = ABCMeta): @@ -96,7 +98,9 @@ def _e_metas(self): yield (None, format_element(x)) def prime_factor(obj): - 'get the primary factor on the `obj`, returns None if none.' + """ + Get the primary factor on the `obj`, returns None if none. + """ f = getattr(obj, '_e_factors', None) if f: return f[0], getattr(obj, f[0], None) @@ -126,7 +130,9 @@ def prime_factors(obj): yield fn, e def format_element(obj, coverage = ()): - 'format the given element with its factors and metadata into a readable string' + """ + Format the given element with its factors and metadata into a readable string. + """ # if it's not an Element, all there is to return is str(obj) if obj in coverage: raise RecursiveFactor(coverage) From 72841f1c9eb81b4017f9a0bd0545ba94edeb8a6f Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sun, 6 Dec 2020 13:01:39 -0700 Subject: [PATCH 058/109] Require 3.3 minimum and update collections references to collections.abc per deprecation warnings. * Clean abc usage by preferring register over multiple inheritance. * Eliminate more types-as-docs * Relocate api.Connection.query to api.Database.query. postgresql.api has been somewhat neglected and should be given a thorough evaluation. --- postgresql/api.py | 139 +++++++----------- postgresql/documentation/changes-v1.3.rst | 2 + postgresql/documentation/clientparameters.rst | 2 +- postgresql/documentation/cluster.rst | 2 +- postgresql/documentation/copyman.rst | 2 +- postgresql/documentation/driver.rst | 20 +-- postgresql/python/itertools.py | 4 +- postgresql/release/distutils.py | 1 + 8 files changed, 68 insertions(+), 104 deletions(-) diff --git a/postgresql/api.py b/postgresql/api.py index 1aa87966..4480bd40 100644 --- a/postgresql/api.py +++ b/postgresql/api.py @@ -11,7 +11,7 @@ This module is used to define "PG-API". It creates a set of ABCs that makes up the basic interfaces used to work with a PostgreSQL server. """ -import collections +import collections.abc import abc from .python.element import Element @@ -125,7 +125,8 @@ class Result(Element): @abc.abstractmethod def close(self) -> None: """ - Close the Result handle. + Close the Result discarding any supporting resources and causing + future read operations to emit empty record sets. """ @property @@ -202,18 +203,12 @@ def statement(self) -> ("Statement", None): `postgresql.api.Database.cursor_from_id`. """ -class Chunks( - Result, - collections.Iterator, - collections.Iterable, -): +@collections.abc.Iterator.register +class Chunks(Result): pass -class Cursor( - Result, - collections.Iterator, - collections.Iterable, -): +@collections.abc.Iterator.register +class Cursor(Result): """ A `Cursor` object is an interface to a sequence of tuples(rows). A result set. Cursors publish a file-like interface for reading tuples from a cursor @@ -259,10 +254,7 @@ def direction(self) -> bool: """ @abc.abstractmethod - def read(self, - quantity : "Number of rows to read" = None, - direction : "Direction to fetch in, defaults to `self.direction`" = None, - ) -> ["Row"]: + def read(self, quantity = None, direction = None) -> ["Row"]: """ Read, fetch, the specified number of rows and return them in a list. If quantity is `None`, all records will be fetched. @@ -312,7 +304,7 @@ class Execution(metaclass = abc.ABCMeta): """ @abc.abstractmethod - def __call__(self, *parameters : "Positional Parameters") -> ["Row"]: + def __call__(self, *parameters) -> ["Row"]: """ Execute the prepared statement with the given arguments as parameters. @@ -324,7 +316,7 @@ def __call__(self, *parameters : "Positional Parameters") -> ["Row"]: """ @abc.abstractmethod - def column(self, *parameters) -> collections.Iterable: + def column(self, *parameters) -> collections.abc.Iterable: """ Return an iterator producing the values of first column of the rows produced by the cursor created from the statement bound with the @@ -345,7 +337,7 @@ def column(self, *parameters) -> collections.Iterable: """ @abc.abstractmethod - def chunks(self, *parameters) -> collections.Iterable: + def chunks(self, *parameters) -> collections.abc.Iterable: """ Return an iterator producing sequences of rows produced by the cursor created from the statement bound with the given parameters. @@ -359,12 +351,12 @@ def chunks(self, *parameters) -> collections.Iterable: Each iteration returns sequences of rows *normally* of length(seq) == chunksize. If chunksize is unspecified, a default, positive integer will be filled in. The rows contained in the sequences are only required to - support the basic `collections.Sequence` interfaces; simple and quick + support the basic `collections.abc.Sequence` interfaces; simple and quick sequence types should be used. """ @abc.abstractmethod - def rows(self, *parameters) -> collections.Iterable: + def rows(self, *parameters) -> collections.abc.Iterable: """ Return an iterator producing rows produced by the cursor created from the statement bound with the given parameters. @@ -382,7 +374,7 @@ def rows(self, *parameters) -> collections.Iterable: """ @abc.abstractmethod - def column(self, *parameters) -> collections.Iterable: + def column(self, *parameters) -> collections.abc.Iterable: """ Return an iterator producing the values of the first column in the cursor created from the statement bound with the given parameters. @@ -407,7 +399,7 @@ def declare(self, *parameters) -> Cursor: """ @abc.abstractmethod - def first(self, *parameters) -> "'First' object that is returned by the query": + def first(self, *parameters): """ Execute the prepared statement with the given arguments as parameters. If the statement returns rows with multiple columns, return the first @@ -426,9 +418,7 @@ def first(self, *parameters) -> "'First' object that is returned by the query": """ @abc.abstractmethod - def load_rows(self, - iterable : "A iterable of tuples to execute the statement with" - ): + def load_rows(self, iterable): """ Given an iterable, `iterable`, feed the produced parameters to the query. This is a bulk-loading interface for parameterized queries. @@ -445,9 +435,7 @@ def load_rows(self, """ @abc.abstractmethod - def load_chunks(self, - iterable : "A iterable of chunks of tuples to execute the statement with" - ): + def load_chunks(self, iterable): """ Given an iterable, `iterable`, feed the produced parameters of the chunks produced by the iterable to the query. This is a bulk-loading interface @@ -465,11 +453,10 @@ def load_chunks(self, that the operation can be optimized. """ -class Statement( - Element, - collections.Callable, - collections.Iterable, -): +@collections.abc.Iterator.register +@collections.abc.Callable.register +@Execution.register +class Statement(Element): """ Instances of `Statement` are returned by the `prepare` method of `Database` instances. @@ -595,13 +582,10 @@ def close(self) -> None: """ Close the prepared statement releasing resources associated with it. """ -Execution.register(Statement) PreparedStatement = Statement -class StoredProcedure( - Element, - collections.Callable, -): +@collections.abc.Callable.register +class StoredProcedure(Element): """ A function stored on the database. """ @@ -609,7 +593,7 @@ class StoredProcedure( _e_factors = ('database',) @abc.abstractmethod - def __call__(self, *args, **kw) -> (object, Cursor, collections.Iterable): + def __call__(self, *args, **kw) -> (object, Cursor, collections.abc.Iterable): """ Execute the procedure with the given arguments. If keyword arguments are passed they must be mapped to the argument whose name matches the key. @@ -759,10 +743,8 @@ def __exit__(self, typ, obj, tb): block's exit. """ -class Settings( - Element, - collections.MutableMapping -): +@collections.abc.MutableMapping.register +class Settings(Element): """ A mapping interface to the session's settings. This provides a direct interface to ``SHOW`` or ``SET`` commands. Identifiers and values need @@ -881,10 +863,7 @@ def client_port(self) -> (int, None): @property @abc.abstractmethod - def xact(self, - isolation : "ISOLATION LEVEL to use with the transaction" = None, - mode : "Mode of the transaction, READ ONLY or READ WRITE" = None, - ) -> Transaction: + def xact(self, isolation = None, mode = None) -> Transaction: """ Create a `Transaction` object using the given keyword arguments as its configuration. @@ -926,9 +905,14 @@ def prepare(self, sql : str) -> Statement: """ @abc.abstractmethod - def statement_from_id(self, - statement_id : "The statement's identification string.", - ) -> Statement: + def query(self, sql : str, *args) -> Execution: + """ + Prepare and execute the statement, `sql`, with the given arguments. + Equivalent to ``db.prepare(sql)(*args)``. + """ + + @abc.abstractmethod + def statement_from_id(self, statement_id) -> Statement: """ Create a `Statement` object that was already prepared on the server. The distinction between this and a regular query is that it @@ -938,9 +922,7 @@ def statement_from_id(self, """ @abc.abstractmethod - def cursor_from_id(self, - cursor_id : "The cursor's identification string." - ) -> Cursor: + def cursor_from_id(self, cursor_id) -> Cursor: """ Create a `Cursor` object from the given `cursor_id` that was already declared on the server. @@ -953,10 +935,7 @@ def cursor_from_id(self, """ @abc.abstractmethod - def proc(self, - procedure_id : \ - "The procedure identifier; a valid ``regprocedure`` or Oid." - ) -> StoredProcedure: + def proc(self, procedure_id) -> StoredProcedure: """ Create a `StoredProcedure` instance using the given identifier. @@ -1030,7 +1009,7 @@ def listening_channels(self) -> ["channel name", ...]: """ @abc.abstractmethod - def iternotifies(self, timeout = None) -> collections.Iterator: + def iternotifies(self, timeout = None) -> collections.abc.Iterator: """ Return an iterator to the notifications received by the connection. The iterator *must* produce triples in the form ``(channel, payload, pid)``. @@ -1096,7 +1075,7 @@ def fatal_exception_message(self, err : Exception) -> (str, None): """ @abc.abstractmethod - def socket_secure(self, socket : "socket object") -> "secured socket": + def socket_secure(self, socket): """ Return a reference to the secured socket using the given parameters. @@ -1106,7 +1085,7 @@ def socket_secure(self, socket : "socket object") -> "secured socket": """ @abc.abstractmethod - def socket_factory_sequence(self) -> [collections.Callable]: + def socket_factory_sequence(self) -> [collections.abc.Callable]: """ Return a sequence of `SocketCreator`s that `Connection` objects will use to create the socket object. @@ -1145,7 +1124,7 @@ def __call__(self, *args, **kw): return self.driver.connection(self, *args, **kw) def __init__(self, - user : "required keyword specifying the user name(str)" = None, + user : str = None, password : str = None, database : str = None, settings : (dict, [(str,str)]) = None, @@ -1179,15 +1158,6 @@ def connector(self) -> Connector: communication and initialization. """ - @property - @abc.abstractmethod - def query(self) -> Execution: - """ - The :py:class:`Execution` instance providing a one-shot query interface:: - - connection.query.(sql, *parameters) == connection.prepare(sql).(*parameters) - """ - @property @abc.abstractmethod def closed(self) -> bool: @@ -1317,18 +1287,13 @@ def data_directory(self) -> str: @abc.abstractmethod def init(self, - initdb : "path to the initdb to use" = None, - user : "name of the cluster's superuser" = None, - password : "superuser's password" = None, - encoding : "the encoding to use for the cluster" = None, - locale : "the locale to use for the cluster" = None, - collate : "the collation to use for the cluster" = None, - ctype : "the ctype to use for the cluster" = None, - monetary : "the monetary to use for the cluster" = None, - numeric : "the numeric to use for the cluster" = None, - time : "the time to use for the cluster" = None, - text_search_config : "default text search configuration" = None, - xlogdir : "location for the transaction log directory" = None, + initdb = None, + user = None, password = None, + encoding = None, locale = None, + collate = None, ctype = None, + monetary = None, numeric = None, time = None, + text_search_config = None, + xlogdir = None, ): """ Create the cluster at the `data_directory` associated with the Cluster @@ -1366,9 +1331,7 @@ def restart(self): """ @abc.abstractmethod - def wait_until_started(self, - timeout : "maximum time to wait" = 10 - ): + def wait_until_started(self, timeout = 10): """ After the start() method is ran, the database may not be ready for use. This method provides a mechanism to block until the cluster is ready for @@ -1379,9 +1342,7 @@ def wait_until_started(self, """ @abc.abstractmethod - def wait_until_stopped(self, - timeout : "maximum time to wait" = 10 - ): + def wait_until_stopped(self, timeout = 10): """ After the stop() method is ran, the database may still be running. This method provides a mechanism to block until the cluster is completely diff --git a/postgresql/documentation/changes-v1.3.rst b/postgresql/documentation/changes-v1.3.rst index 962887c5..d265e6d5 100644 --- a/postgresql/documentation/changes-v1.3.rst +++ b/postgresql/documentation/changes-v1.3.rst @@ -5,3 +5,5 @@ Changes in v1.3 ----- * Commit DB-API 2.0 ClientCannotConnect exception correction. + * Eliminate types-as-documentation annotations. + * Eliminate multiple inheritance in `postgresql.api` in favor of ABC registration. diff --git a/postgresql/documentation/clientparameters.rst b/postgresql/documentation/clientparameters.rst index e85bd675..da367395 100644 --- a/postgresql/documentation/clientparameters.rst +++ b/postgresql/documentation/clientparameters.rst @@ -68,7 +68,7 @@ accept: ``environ`` Environment variables to extract client parameter variables from. - Defaults to `os.environ` and expects a `collections.Mapping` interface. + Defaults to `os.environ` and expects a `collections.abc.Mapping` interface. ``environ_prefix`` Environment variable prefix to use. Defaults to "PG". This allows the diff --git a/postgresql/documentation/cluster.rst b/postgresql/documentation/cluster.rst index 0ba2bc2d..1993ea28 100644 --- a/postgresql/documentation/cluster.rst +++ b/postgresql/documentation/cluster.rst @@ -348,7 +348,7 @@ Methods and properties available on `postgresql.cluster.Cluster` instances: `Cluster.wait_until_started`. ``Cluster.settings`` - A `collections.Mapping` interface to the ``postgresql.conf`` file of the + A `collections.abc.Mapping` interface to the ``postgresql.conf`` file of the cluster. A notable extension to the mapping interface is the ``getset`` method. This diff --git a/postgresql/documentation/copyman.rst b/postgresql/documentation/copyman.rst index 37304937..d4a18cb1 100644 --- a/postgresql/documentation/copyman.rst +++ b/postgresql/documentation/copyman.rst @@ -260,7 +260,7 @@ The following Producers are available: ``postgresql.copyman.StatementProducer(postgresql.api.Statement)`` Given a Statement producing COPY data, construct a Producer. - ``postgresql.copyman.IteratorProducer(collections.Iterator)`` + ``postgresql.copyman.IteratorProducer(collections.abc.Iterator)`` Given an Iterator producing *chunks* of COPY lines, construct a Producer to manage the data coming from the iterator. diff --git a/postgresql/documentation/driver.rst b/postgresql/documentation/driver.rst index b373f29e..00d3aa73 100644 --- a/postgresql/documentation/driver.rst +++ b/postgresql/documentation/driver.rst @@ -319,7 +319,7 @@ The methods and properties on the connection object are ready for use: ``Connection.proc(procedure_id)`` Create a `postgresql.api.StoredProcedure` object referring to a stored procedure on the database. The returned object will provide a - `collections.Callable` interface to the stored procedure on the server. See + `collections.abc.Callable` interface to the stored procedure on the server. See `Stored Procedures`_ for more information. ``Connection.statement_from_id(statement_id)`` @@ -350,7 +350,7 @@ The methods and properties on the connection object are ready for use: information. ``Connection.settings`` - A property providing a `collections.MutableMapping` interface to the + A property providing a `collections.abc.MutableMapping` interface to the database's SQL settings. See `Settings`_ for more information. ``Connection.clone()`` @@ -557,7 +557,7 @@ Prepared statement objects have a few execution methods: ``Statement.chunks(*parameters)`` This access point is designed for situations where rows are being streamed out - quickly. It is a method that returns a ``collections.Iterator`` that produces + quickly. It is a method that returns a ``collections.abc.Iterator`` that produces *sequences* of rows. This is the most efficient way to get rows from the database. The rows in the sequences are ``builtins.tuple`` objects. @@ -569,11 +569,11 @@ Prepared statement objects have a few execution methods: ``Statement.close()`` Close the statement inhibiting further use. - ``Statement.load_rows(collections.Iterable(parameters))`` + ``Statement.load_rows(collections.abc.Iterable(parameters))`` Given an iterable producing parameters, execute the statement for each iteration. Always returns `None`. - ``Statement.load_chunks(collections.Iterable(collections.Iterable(parameters)))`` + ``Statement.load_chunks(collections.abc.Iterable(collections.abc.Iterable(parameters)))`` Given an iterable of iterables producing parameters, execute the statement for each parameter produced. However, send the all execution commands with the corresponding parameters of each chunk before reading any results. @@ -1075,7 +1075,7 @@ critical. Row Interface Points -------------------- -Rows implement the `collections.Mapping` and `collections.Sequence` interfaces. +Rows implement the `collections.abc.Mapping` and `collections.abc.Sequence` interfaces. ``Row.keys()`` An iterable producing the column names. Order is not guaranteed. See the @@ -1214,8 +1214,8 @@ Queries have access to all execution methods: * ``Connection.query.first(sql, *parameters)`` * ``Connection.query.chunks(sql, *parameters)`` * ``Connection.query.declare(sql, *parameters)`` - * ``Connection.query.load_rows(sql, collections.Iterable(parameters))`` - * ``Connection.query.load_chunks(collections.Iterable(collections.Iterable(parameters)))`` + * ``Connection.query.load_rows(sql, collections.abc.Iterable(parameters))`` + * ``Connection.query.load_chunks(collections.abc.Iterable(collections.abc.Iterable(parameters)))`` In cases where a sequence of one-shot queries needs to be performed, it may be important to avoid unnecessary repeat attribute resolution from the connection object as the ``query`` @@ -1461,7 +1461,7 @@ Settings SQL's SHOW and SET provides a means to configure runtime parameters on the database("GUC"s). In order to save the user some grief, a -`collections.MutableMapping` interface is provided to simplify configuration. +`collections.abc.MutableMapping` interface is provided to simplify configuration. The ``settings`` attribute on the connection provides the interface extension. @@ -1485,7 +1485,7 @@ Settings Interface Points ------------------------- Manipulation and interrogation of the connection's settings is achieved by -using the standard `collections.MutableMapping` interfaces. +using the standard `collections.abc.MutableMapping` interfaces. ``Connection.settings[k]`` Get the value of a single setting. diff --git a/postgresql/python/itertools.py b/postgresql/python/itertools.py index 94672367..08fcdb5d 100644 --- a/postgresql/python/itertools.py +++ b/postgresql/python/itertools.py @@ -4,10 +4,10 @@ """ itertools extensions """ -import collections +import collections.abc from itertools import cycle, islice -def interlace(*iters, next = next) -> collections.Iterable: +def interlace(*iters, next = next) -> collections.abc.Iterable: """ interlace(i1, i2, ..., in) -> ( i1-0, i2-0, ..., in-0, diff --git a/postgresql/release/distutils.py b/postgresql/release/distutils.py index 9e018a85..1d414488 100644 --- a/postgresql/release/distutils.py +++ b/postgresql/release/distutils.py @@ -177,6 +177,7 @@ def standard_setup_keywords(build_extensions = True, prefix = default_prefix): 'packages' : list(prefixed_packages(prefix = prefix)), 'package_data' : dict(prefixed_package_data(prefix = prefix)), 'cmdclass': dict(test=TestCommand), + 'python_requires': '>=3.3', } if build_extensions: d['ext_modules'] = list(prefixed_extensions(prefix = prefix)) From 28bf64948bcab9b874610e6c980689d78f38bacc Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sun, 6 Dec 2020 23:01:01 -0700 Subject: [PATCH 059/109] Alter postgresql.temporal to prefer PGTEST environment over PGINSTALLATION. - Allows tests to run against arbitrary PQ servers(docker/podman/etc). - Adjust non-pg_tmp tests to skip if PGINSTALLATION is not available. --- postgresql/documentation/changes-v1.3.rst | 4 + postgresql/temporal.py | 49 ++++++---- postgresql/test/test_cluster.py | 16 ++-- postgresql/test/test_connect.py | 109 +++++++++++++--------- postgresql/test/test_dbapi20.py | 32 +++---- postgresql/test/test_driver.py | 14 ++- postgresql/test/test_ssl_connect.py | 20 ++++ postgresql/test/testall.py | 8 +- 8 files changed, 150 insertions(+), 102 deletions(-) diff --git a/postgresql/documentation/changes-v1.3.rst b/postgresql/documentation/changes-v1.3.rst index d265e6d5..5ec51561 100644 --- a/postgresql/documentation/changes-v1.3.rst +++ b/postgresql/documentation/changes-v1.3.rst @@ -7,3 +7,7 @@ Changes in v1.3 * Commit DB-API 2.0 ClientCannotConnect exception correction. * Eliminate types-as-documentation annotations. * Eliminate multiple inheritance in `postgresql.api` in favor of ABC registration. + * Add support for PGTEST environment variable (pq-IRI) to improve test performance + and to aid in cases where the target fixture is already available. + This should help for testing the driver against servers that are not actually + postgresql. diff --git a/postgresql/temporal.py b/postgresql/temporal.py index 1c128bea..a19a845d 100644 --- a/postgresql/temporal.py +++ b/postgresql/temporal.py @@ -29,8 +29,8 @@ class Temporal(object): Or `pg_tmp` can decorate a method or function. """ - #: Format the cluster directory name. - cluster_dirname = 'pg_tmp_{0}_{1}'.format + format_sandbox_id = staticmethod(('sandbox{0}_{1}').format) + cluster_dirname = staticmethod(('pg_tmp_{0}_{1}').format) cluster = None _init_pid_ = None @@ -91,7 +91,7 @@ def init(self, "environment variable to the `pg_config` path" } ): - if self.cluster is not None: + if self.cluster is not None or 'PGTEST' in os.environ: return ## # Hasn't been created yet, but doesn't matter. @@ -156,7 +156,7 @@ def init(self, unix_socket_directories = cluster.data_directory, )) - # Start it up. + # Start the database cluster. with open(self.logfile, 'w') as lfo: cluster.start(logfile = lfo) cluster.wait_until_started() @@ -165,18 +165,23 @@ def init(self, c = cluster.connection(user = 'test', database = 'template1',) with c: c.execute('create database test') - # It's ready. self.cluster = cluster def push(self): - c = self.cluster.connection(user = 'test') - c.connect() + if 'PGTEST' in os.environ: + from . import open as pg_open + c = pg_open(os.environ['PGTEST']) # Ignoring PGINSTALLATION. + else: + c = self.cluster.connection(user = 'test') + c.connect() + extras = [] + sbid = self.format_sandbox_id(os.getpid(), self.sandbox_id + 1) - def new_pg_tmp_connection(l = extras, c = c, sbid = 'sandbox' + str(self.sandbox_id + 1)): + def new_pg_tmp_connection(l = extras, clone = c.clone, sbid = sbid): # Used to create a new connection that will be closed # when the context stack is popped along with 'db'. - l.append(c.clone()) + l.append(clone()) l[-1].settings['search_path'] = str(sbid) + ',' + l[-1].settings['search_path'] return l[-1] @@ -205,7 +210,7 @@ def new_pg_tmp_connection(l = extras, c = c, sbid = 'sandbox' + str(self.sandbox builtins.__dict__.update(local_builtins) self.sandbox_id += 1 - def pop(self, exc, drop_schema = 'DROP SCHEMA sandbox{0} CASCADE'.format): + def pop(self, exc, drop_schema = ('DROP SCHEMA {0} CASCADE').format): local_builtins, extras = self.builtins_stack.pop() self.sandbox_id -= 1 @@ -235,32 +240,36 @@ def pop(self, exc, drop_schema = 'DROP SCHEMA sandbox{0} CASCADE'.format): # Interrupted and closed all the other connections at this level; # now remove the sandbox schema. - c = self.cluster.connection(user = 'test') - with c: + xdb = local_builtins['db'] + with xdb.clone() as c: # Use a new connection so that the state of # the context connection will not have to be # contended with. - c.execute(drop_schema(self.sandbox_id+1)) + c.execute(drop_schema(self.format_sandbox_id(os.getpid(), self.sandbox_id + 1))) else: - # interrupt + # interrupt exception; avoid waiting for close pass + def _init_c(self, cxn): + cxn.connect() + sb = self.format_sandbox_id(os.getpid(), self.sandbox_id) + cxn.execute('CREATE SCHEMA ' + sb) + cxn.settings['search_path'] = ','.join((sb, cxn.settings['search_path'])) + def __enter__(self): if self.cluster is None: self.init() + self.push() try: - db.connect() - db.execute('CREATE SCHEMA sandbox' + str(self.sandbox_id)) - db.settings['search_path'] = 'sandbox' + str(self.sandbox_id) + ',' + db.settings['search_path'] + self._init_c(builtins.db) except Exception as e: # failed to initialize sandbox schema; pop it. self.pop(e) raise def __exit__(self, exc, val, tb): - if self.cluster is not None: - self.pop(val) + self.pop(val) -#: The process' temporary cluster. +#: The process' temporary cluster or connection source. pg_tmp = Temporal() diff --git a/postgresql/test/test_cluster.py b/postgresql/test/test_cluster.py index b4aee97a..027b5fda 100644 --- a/postgresql/test/test_cluster.py +++ b/postgresql/test/test_cluster.py @@ -9,19 +9,16 @@ from .. import installation from ..cluster import Cluster, ClusterStartupError -default_install = installation.default() -if default_install is None: - sys.stderr.write("ERROR: cannot find 'default' pg_config\n") - sys.stderr.write("HINT: set the PGINSTALLATION environment variable to the `pg_config` path\n") - sys.exit(1) +default_installation = installation.default() class test_cluster(unittest.TestCase): def setUp(self): - self.cluster = Cluster(default_install, 'test_cluster',) + self.cluster = Cluster(default_installation, 'test_cluster',) def tearDown(self): - self.cluster.drop() - self.cluster = None + if self.cluster.installation is not None: + self.cluster.drop() + self.cluster = None def start_cluster(self, logfile = None): self.cluster.start(logfile = logfile) @@ -46,6 +43,7 @@ def init(self, *args, **kw): usd : self.cluster.data_directory, }) + @unittest.skipIf(default_installation is None, "no installation provided by environment") def testSilentMode(self): self.init() self.cluster.settings['silent_mode'] = 'on' @@ -66,6 +64,7 @@ def testSilentMode(self): elif self.cluster.installation.version_info[:2] >= (9, 2): self.fail("silent_mode unexpectedly supported on PostgreSQL >=9.2") + @unittest.skipIf(default_installation is None, "no installation provided by environment") def testSuperPassword(self): self.init( user = 'test', @@ -81,6 +80,7 @@ def testSuperPassword(self): with c: self.assertEqual(c.prepare('select 1').first(), 1) + @unittest.skipIf(default_installation is None, "no installation provided by environment") def testNoParameters(self): """ Simple init and drop. diff --git a/postgresql/test/test_connect.py b/postgresql/test/test_connect.py index 4e29b956..ed587956 100644 --- a/postgresql/test/test_connect.py +++ b/postgresql/test/test_connect.py @@ -18,6 +18,7 @@ from .. import driver as pg_driver from .. import open as pg_open +default_installation = installation.default() def check_for_ipv6(): result = False @@ -47,28 +48,25 @@ class TestCaseWithCluster(unittest.TestCase): """ postgresql.driver *interface* tests. """ + installation = default_installation + def __init__(self, *args, **kw): super().__init__(*args, **kw) - self.installation = installation.default() self.cluster_path = \ 'pypg_test_' \ + str(os.getpid()) + getattr(self, 'cluster_path_suffix', '') - if self.installation is None: - sys.stderr.write("ERROR: cannot find 'default' pg_config\n") - sys.stderr.write( - "HINT: set the PGINSTALLATION environment variable to the `pg_config` path\n" - ) - sys.exit(1) - self.cluster = pg_cluster.Cluster( self.installation, self.cluster_path, ) - if self.cluster.initialized(): - self.cluster.drop() - self.disable_replication = self.installation.version_info[:2] > (9, 6) + @property + def disable_replication(self): + """ + Whether replication settings should be disabled. + """ + return self.installation.version_info[:2] > (9, 6) def configure_cluster(self): self.cluster_port = find_available_port() @@ -126,25 +124,33 @@ def initialize_database(self): def connection(self, *args, **kw): return self.cluster.connection(*args, user = 'test', **kw) + def drop_cluster(self): + if self.cluster.initialized(): + self.cluster.drop() + def run(self, *args, **kw): - if not self.cluster.initialized(): - self.cluster.encoding = 'utf-8' - self.cluster.init( - user = 'test', - encoding = self.cluster.encoding, - logfile = None, - ) - sys.stderr.write('*') - try: - atexit.register(self.cluster.drop) - self.configure_cluster() - self.cluster.start(logfile = sys.stdout) - self.cluster.wait_until_started() - self.initialize_database() - except Exception: - self.cluster.drop() - atexit.unregister(self.cluster.drop) - raise + if 'PGINSTALLATION' not in os.environ: + # Expect tests to show skipped. + return super().run(*args, **kw) + + # From prior test run? + if self.cluster.initialized(): + self.cluster.drop() + + self.cluster.encoding = 'utf-8' + self.cluster.init( + user = 'test', + encoding = self.cluster.encoding, + logfile = None, + ) + sys.stderr.write('*') + + atexit.register(self.drop_cluster) + self.configure_cluster() + self.cluster.start(logfile = sys.stdout) + self.cluster.wait_until_started() + self.initialize_database() + if not self.cluster.running(): self.cluster.start() self.cluster.wait_until_started() @@ -157,7 +163,7 @@ def run(self, *args, **kw): class test_connect(TestCaseWithCluster): """ - postgresql.driver connectivity tests + postgresql.driver connection tests """ ip6 = '::1' ip4 = '127.0.0.1' @@ -179,9 +185,10 @@ class test_connect(TestCaseWithCluster): def __init__(self, *args, **kw): super().__init__(*args,**kw) - # 8.4 nixed this. - vi = self.cluster.installation.version_info - self.check_crypt_user = (vi < (8,4)) + + @property + def check_crypt_user(self): + return (self.cluster.installation.version_info < (8,4)) def configure_cluster(self): super().configure_cluster() @@ -220,6 +227,7 @@ def initialize_database(self): if self.check_crypt_user: db.execute(self.mk_crypt_user) + @unittest.skipIf(default_installation is None, "no installation provided by environment") def test_pg_open_SQL_ASCII(self): # postgresql.open host, port = self.cluster.address() @@ -232,6 +240,7 @@ def test_pg_open_SQL_ASCII(self): self.assertEqual(db.settings['client_encoding'], 'SQL_ASCII') self.assertTrue(db.closed) + @unittest.skipIf(default_installation is None, "no installation provided by environment") def test_pg_open_keywords(self): host, port = self.cluster.address() # straight test, no IRI @@ -273,6 +282,7 @@ def test_pg_open_keywords(self): self.assertEqual(db.prepare('select 1')(), [(1,)]) self.assertEqual(db.settings['search_path'], 'public') + @unittest.skipIf(default_installation is None, "no installation provided by environment") def test_pg_open(self): # postgresql.open host, port = self.cluster.address() @@ -355,6 +365,7 @@ def test_pg_open(self): if os.path.exists('pg_service.conf'): os.remove('pg_service.conf') + @unittest.skipIf(default_installation is None, "no installation provided by environment") def test_dbapi_connect(self): host, port = self.cluster.address() MD5 = dbapi20.connect( @@ -410,6 +421,7 @@ def test_dbapi_connect(self): TRUST.cursor().execute, 'select 1' ) + @unittest.skipIf(default_installation is None, "no installation provided by environment") def test_dbapi_connect_failure(self): host, port = self.cluster.address() badlogin = (lambda: dbapi20.connect( @@ -421,6 +433,7 @@ def test_dbapi_connect_failure(self): )) self.assertRaises(pg_exc.ClientCannotConnectError, badlogin) + @unittest.skipIf(default_installation is None, "no installation provided by environment") def test_IP4_connect(self): C = pg_driver.default.ip4( user = 'test', @@ -432,18 +445,20 @@ def test_IP4_connect(self): with C() as c: self.assertEqual(c.prepare('select 1').first(), 1) - if has_ipv6: - def test_IP6_connect(self): - C = pg_driver.default.ip6( - user = 'test', - host = '::1', - database = 'test', - port = self.cluster.address()[1], - **self.params - ) - with C() as c: - self.assertEqual(c.prepare('select 1').first(), 1) + @unittest.skipIf(default_installation is None, "no installation provided by environment") + @unittest.skipIf(not has_ipv6, "platform may not support IPv6") + def test_IP6_connect(self): + C = pg_driver.default.ip6( + user = 'test', + host = '::1', + database = 'test', + port = self.cluster.address()[1], + **self.params + ) + with C() as c: + self.assertEqual(c.prepare('select 1').first(), 1) + @unittest.skipIf(default_installation is None, "no installation provided by environment") def test_Host_connect(self): C = pg_driver.default.host( user = 'test', @@ -455,6 +470,7 @@ def test_Host_connect(self): with C() as c: self.assertEqual(c.prepare('select 1').first(), 1) + @unittest.skipIf(default_installation is None, "no installation provided by environment") def test_md5_connect(self): c = self.cluster.connection( user = 'md5', @@ -465,6 +481,7 @@ def test_md5_connect(self): with c: self.assertEqual(c.prepare('select current_user').first(), 'md5') + @unittest.skipIf(default_installation is None, "no installation provided by environment") def test_crypt_connect(self): if self.check_crypt_user: c = self.cluster.connection( @@ -476,6 +493,7 @@ def test_crypt_connect(self): with c: self.assertEqual(c.prepare('select current_user').first(), 'crypt') + @unittest.skipIf(default_installation is None, "no installation provided by environment") def test_password_connect(self): c = self.cluster.connection( user = 'password', @@ -485,6 +503,7 @@ def test_password_connect(self): with c: self.assertEqual(c.prepare('select current_user').first(), 'password') + @unittest.skipIf(default_installation is None, "no installation provided by environment") def test_trusted_connect(self): c = self.cluster.connection( user = 'trusted', @@ -495,6 +514,7 @@ def test_trusted_connect(self): with c: self.assertEqual(c.prepare('select current_user').first(), 'trusted') + @unittest.skipIf(default_installation is None, "no installation provided by environment") def test_Unix_connect(self): if not has_unix_sock: return @@ -510,6 +530,7 @@ def test_Unix_connect(self): self.assertEqual(c.prepare('select 1').first(), 1) self.assertEqual(c.client_address, None) + @unittest.skipIf(default_installation is None, "no installation provided by environment") def test_pg_open_unix(self): if not has_unix_sock: return diff --git a/postgresql/test/test_dbapi20.py b/postgresql/test/test_dbapi20.py index 3cbd6626..6a0f2380 100644 --- a/postgresql/test/test_dbapi20.py +++ b/postgresql/test/test_dbapi20.py @@ -93,27 +93,21 @@ def executeDDL1(self,cursor): def executeDDL2(self,cursor): cursor.execute(self.ddl2) + def setUp(self): + pg_tmp.init() + pg_tmp.push() + pg_tmp._init_c(db) + def tearDown(self): - con = self._connect() - try: - cur = con.cursor() - for ddl in (self.xddl1, self.xddl2): - try: - cur.execute(ddl) - con.commit() - except self.driver.Error: - # Assume table didn't exist. Other tests will check if - # execute is busted. - pass - finally: - con.close() + pg_tmp.pop(None) def _connect(self): - pg_tmp.init() - host, port = pg_tmp.cluster.address() - return self.driver.connect( - user = 'test', host = host, port = port, - ) + c = db.clone() + c.__class__ = self.driver.Connection + c._xact = c.xact() + c._xact.start() + c._dbapi_connected_flag = True + return c def test_connect(self): con = self._connect() @@ -708,7 +702,7 @@ def test_mixedfetch(self): def help_nextset_setUp(self,cur): ''' Should create a procedure called deleteme - that returns two result sets, first the + that returns two result sets, first the number of rows in booze then "name from booze" ''' cur.execute('select name from ' + self.booze_name) diff --git a/postgresql/test/test_driver.py b/postgresql/test/test_driver.py index 62740e46..f590bf12 100644 --- a/postgresql/test/test_driver.py +++ b/postgresql/test/test_driver.py @@ -538,12 +538,16 @@ def testStatementAndCursorMetadata(self): self.assertEqual(tuple(c.pg_column_types), (pg_types.TEXTOID, pg_types.VARCHAROID)) self.assertEqual(tuple(c.column_types), (str,str)) - db.execute("CREATE TYPE public.myudt AS (i int)") + # Should be pg_temp or sandbox. + schema = db.settings['search_path'].split(',')[0] + typpath = '"%s"."myudt"' %(schema,) + + db.execute("CREATE TYPE myudt AS (i int)") myudt_oid = db.prepare("select oid from pg_type WHERE typname='myudt'").first() - ps = db.prepare("SELECT $1::text AS my_column1, $2::varchar AS my_column2, $3::public.myudt AS my_column3") + ps = db.prepare("SELECT $1::text AS my_column1, $2::varchar AS my_column2, $3::myudt AS my_column3") self.assertEqual(tuple(ps.column_names), ('my_column1','my_column2', 'my_column3')) - self.assertEqual(tuple(ps.sql_column_types), ('pg_catalog.text', 'CHARACTER VARYING', '"public"."myudt"')) - self.assertEqual(tuple(ps.sql_parameter_types), ('pg_catalog.text', 'CHARACTER VARYING', '"public"."myudt"')) + self.assertEqual(tuple(ps.sql_column_types), ('pg_catalog.text', 'CHARACTER VARYING', typpath)) + self.assertEqual(tuple(ps.sql_parameter_types), ('pg_catalog.text', 'CHARACTER VARYING', typpath)) self.assertEqual(tuple(ps.pg_column_types), ( pg_types.TEXTOID, pg_types.VARCHAROID, myudt_oid) ) @@ -554,7 +558,7 @@ def testStatementAndCursorMetadata(self): self.assertEqual(tuple(ps.column_types), (str,str,tuple)) c = ps.declare('textdata', 'varchardata', (123,)) self.assertEqual(tuple(c.column_names), ('my_column1','my_column2', 'my_column3')) - self.assertEqual(tuple(c.sql_column_types), ('pg_catalog.text', 'CHARACTER VARYING', '"public"."myudt"')) + self.assertEqual(tuple(c.sql_column_types), ('pg_catalog.text', 'CHARACTER VARYING', typpath)) self.assertEqual(tuple(c.pg_column_types), ( pg_types.TEXTOID, pg_types.VARCHAROID, myudt_oid )) diff --git a/postgresql/test/test_ssl_connect.py b/postgresql/test/test_ssl_connect.py index ce2e3e2c..fc72c86c 100644 --- a/postgresql/test/test_ssl_connect.py +++ b/postgresql/test/test_ssl_connect.py @@ -10,6 +10,12 @@ from ..driver import dbapi20 from . import test_connect +default_installation = test_connect.default_installation + +has_ssl = False +if default_installation is not None: + has_ssl = default_installation.ssl + server_key = """ -----BEGIN RSA PRIVATE KEY----- MIICXAIBAAKBgQCy8veVaqL6MZVT8o0j98ggZYfibGwSN4XGC4rfineA2QZhi8t+ @@ -104,6 +110,9 @@ class test_ssl_connect(test_connect.test_connect): cluster_path_suffix = '_test_ssl_connect' def configure_cluster(self): + if not has_ssl: + return + super().configure_cluster() self.cluster.settings['ssl'] = 'on' with open(self.cluster.hba_file, 'a') as hba: @@ -126,6 +135,9 @@ def configure_cluster(self): os.chmod(crt_file, 0o700) def initialize_database(self): + if not has_ssl: + return + super().initialize_database() with self.cluster.connection(user = 'test') as db: db.execute( @@ -135,6 +147,8 @@ def initialize_database(self): """ ) + @unittest.skipIf(default_installation is None, "no installation provided by environment") + @unittest.skipIf(not has_ssl, "could not detect installation tls") def test_ssl_mode_require(self): host, port = self.cluster.address() params = dict(self.params) @@ -167,6 +181,8 @@ def test_ssl_mode_require(self): self.assertEqual(c.prepare('select 1').first(), 1) self.assertEqual(c.security, 'ssl') + @unittest.skipIf(default_installation is None, "no installation provided by environment") + @unittest.skipIf(not has_ssl, "could not detect installation tls") def test_ssl_mode_disable(self): host, port = self.cluster.address() params = dict(self.params) @@ -200,6 +216,8 @@ def test_ssl_mode_disable(self): self.assertEqual(c.prepare('select 1').first(), 1) self.assertEqual(c.security, None) + @unittest.skipIf(default_installation is None, "no installation provided by environment") + @unittest.skipIf(not has_ssl, "could not detect installation tls") def test_ssl_mode_prefer(self): host, port = self.cluster.address() params = dict(self.params) @@ -233,6 +251,8 @@ def test_ssl_mode_prefer(self): self.assertEqual(c.prepare('select 1').first(), 1) self.assertEqual(c.security, None) + @unittest.skipIf(default_installation is None, "no installation provided by environment") + @unittest.skipIf(not has_ssl, "could not detect installation tls") def test_ssl_mode_allow(self): host, port = self.cluster.address() params = dict(self.params) diff --git a/postgresql/test/testall.py b/postgresql/test/testall.py index 366d3ad4..b32ccaa1 100644 --- a/postgresql/test/testall.py +++ b/postgresql/test/testall.py @@ -17,13 +17,9 @@ from .test_installation import * from .test_cluster import * -# These two require custom cluster configurations. +# Expects PGINSTALLATION to be set. Tests may be skipped. from .test_connect import * -# No SSL? cluster initialization will fail. -if default().ssl: - from .test_ssl_connect import * -else: - stderr.write("NOTICE: installation doesn't support SSL\n") +from .test_ssl_connect import * try: from .test_optimized import * From 094224d6cb38985ac7562d829ec85200eb9344bd Mon Sep 17 00:00:00 2001 From: James William Pye Date: Mon, 7 Dec 2020 12:29:16 -0700 Subject: [PATCH 060/109] Use a more descriptive value for the user parameter. --- postgresql/clientparameters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/clientparameters.py b/postgresql/clientparameters.py index 0f69d6fb..fb5046d3 100644 --- a/postgresql/clientparameters.py +++ b/postgresql/clientparameters.py @@ -20,7 +20,7 @@ to support sub-dictionaries like settings:: >>> normal_params = { - 'user' : 'jwp', + 'user' : 'dbusername', 'host' : 'localhost', 'settings' : {'default_statistics_target' : 200, 'search_path' : 'home,public'} } From 2c48812910d067bc0bc2730c3ca3279885bb82d3 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Mon, 7 Dec 2020 12:49:13 -0700 Subject: [PATCH 061/109] Remove other cases of developer initials in documentation. This should improve clarity in cases where where a reader is not familar with the author's initials. :) --- postgresql/documentation/bin.rst | 16 ++++++++-------- postgresql/documentation/clientparameters.rst | 14 +++++++------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/postgresql/documentation/bin.rst b/postgresql/documentation/bin.rst index 31de6680..43e7a76e 100644 --- a/postgresql/documentation/bin.rst +++ b/postgresql/documentation/bin.rst @@ -86,17 +86,17 @@ pg_python Examples Module execution taking advantage of the new built-ins:: $ python3 -m postgresql.bin.pg_python -h localhost -W -m timeit "prepare('SELECT 1').first()" - Password for pg_python[pq://jwp@localhost:5432]: + Password for pg_python[pq://dbusername@localhost:5432]: 1000 loops, best of 3: 1.35 msec per loop $ python3 -m postgresql.bin.pg_python -h localhost -W -m timeit -s "ps=prepare('SELECT 1')" "ps.first()" - Password for pg_python[pq://jwp@localhost:5432]: + Password for pg_python[pq://dbusername@localhost:5432]: 1000 loops, best of 3: 442 usec per loop Simple interactive usage:: $ python3 -m postgresql.bin.pg_python -h localhost -W - Password for pg_python[pq://jwp@localhost:5432]: + Password for pg_python[pq://dbusername@localhost:5432]: >>> ps = prepare('select 1') >>> ps.first() 1 @@ -142,22 +142,22 @@ Examples Modifying a simple configuration file:: $ echo "setting = value" >pg.conf - + # change 'setting' $ python3 -m postgresql.bin.pg_dotconf pg.conf setting=newvalue - + $ cat pg.conf setting = 'newvalue' - + # new settings are appended to the file $ python3 -m postgresql.bin.pg_dotconf pg.conf another_setting=value $ cat pg.conf setting = 'newvalue' another_setting = 'value' - + # comment a setting $ python3 -m postgresql.bin.pg_dotconf pg.conf another_setting - + $ cat pg.conf setting = 'newvalue' #another_setting = 'value' diff --git a/postgresql/documentation/clientparameters.rst b/postgresql/documentation/clientparameters.rst index da367395..048fa75f 100644 --- a/postgresql/documentation/clientparameters.rst +++ b/postgresql/documentation/clientparameters.rst @@ -100,16 +100,16 @@ instructed to do by the ``prompt_password`` key in the parameters:: >>> import postgresql.clientparameters as pg_param >>> p = pg_param.collect(prompt_title = 'my_prompt!', parameters = {'prompt_password':True}) - Password for my_prompt![pq://jwp@localhost:5432]: + Password for my_prompt![pq://dbusername@localhost:5432]: >>> p - {'host': 'localhost', 'user': 'jwp', 'password': 'secret', 'port': 5432} + {'host': 'localhost', 'user': 'dbusername', 'password': 'secret', 'port': 5432} If `None`, it will leave the necessary password resolution information in the parameters dictionary for ``resolve_password``:: >>> p = pg_param.collect(prompt_title = None, parameters = {'prompt_password':True}) >>> p - {'pgpassfile': '/Users/jwp/.pgpass', 'prompt_password': True, 'host': 'localhost', 'user': 'jwp', 'port': 5432} + {'pgpassfile': '/home/{USER}/.pgpass', 'prompt_password': True, 'host': 'localhost', 'user': 'dbusername', 'port': 5432} Of course, ``'prompt_password'`` is normally specified when ``parsed_options`` received a ``-W`` option from the command line:: @@ -118,9 +118,9 @@ received a ``-W`` option from the command line:: >>> co, ca = op.parse_args(['-W']) >>> p = pg_param.collect(parsed_options = co) >>> p=pg_param.collect(parsed_options = co) - Password for [pq://jwp@localhost:5432]: + Password for [pq://dbusername@localhost:5432]: >>> p - {'host': 'localhost', 'user': 'jwp', 'password': 'secret', 'port': 5432} + {'host': 'localhost', 'user': 'dbusername', 'password': 'secret', 'port': 5432} >>> @@ -166,10 +166,10 @@ When resolution occurs, the ``prompt_password``, ``prompt_title``, and >>> p=pg_param.collect(prompt_title = None) >>> p - {'pgpassfile': '/Users/jwp/.pgpass', 'host': 'localhost', 'user': 'jwp', 'port': 5432} + {'pgpassfile': '/Users/{USER}/.pgpass', 'host': 'localhost', 'user': 'dbusername', 'port': 5432} >>> pg_param.resolve_password(p) >>> p - {'host': 'localhost', 'password': 'secret', 'user': 'jwp', 'port': 5432} + {'host': 'localhost', 'password': 'secret', 'user': 'dbusername', 'port': 5432} Defaults From 78af481dfb712ab9bb299907134c2a65d40e36c6 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Mon, 7 Dec 2020 13:10:49 -0700 Subject: [PATCH 062/109] Use the traditional location in the example. --- postgresql/documentation/clientparameters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/documentation/clientparameters.rst b/postgresql/documentation/clientparameters.rst index 048fa75f..8c8441cf 100644 --- a/postgresql/documentation/clientparameters.rst +++ b/postgresql/documentation/clientparameters.rst @@ -166,7 +166,7 @@ When resolution occurs, the ``prompt_password``, ``prompt_title``, and >>> p=pg_param.collect(prompt_title = None) >>> p - {'pgpassfile': '/Users/{USER}/.pgpass', 'host': 'localhost', 'user': 'dbusername', 'port': 5432} + {'pgpassfile': '/home/{USER}/.pgpass', 'host': 'localhost', 'user': 'dbusername', 'port': 5432} >>> pg_param.resolve_password(p) >>> p {'host': 'localhost', 'password': 'secret', 'user': 'dbusername', 'port': 5432} From afbe2a04e207889c2c8d8342e06a572ac83139b0 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Mon, 7 Dec 2020 21:36:53 -0700 Subject: [PATCH 063/109] Correct inappropriate NotImplementedError usage. --- postgresql/clientparameters.py | 2 +- postgresql/types/geometry.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/postgresql/clientparameters.py b/postgresql/clientparameters.py index fb5046d3..55409767 100644 --- a/postgresql/clientparameters.py +++ b/postgresql/clientparameters.py @@ -505,7 +505,7 @@ def x_pg_service(service_name, config): ) def x_pg_ldap(ldap_url, config): - raise NotImplementedError("cannot resolve ldap URLs: " + str(ldap_url)) + raise Exception("cannot resolve ldap URLs") default_x_callbacks = { 'settings' : x_settings, diff --git a/postgresql/types/geometry.py b/postgresql/types/geometry.py index b1ed9f89..000a8845 100644 --- a/postgresql/types/geometry.py +++ b/postgresql/types/geometry.py @@ -95,9 +95,6 @@ def __str__(self): def parallel(self, ob): return self.slope == type(self)(ob).slope - def intersect(self, ob): - raise NotImplementedError - def perpendicular(self, ob): return (self.slope / type(self)(ob).slope) == -1.0 From b6db7b617bf2b3299907cf09179b772a73a53ad1 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Mon, 7 Dec 2020 21:38:11 -0700 Subject: [PATCH 064/109] Remove superfluous whitespace and capitalize doc-string sentence. --- postgresql/types/geometry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/postgresql/types/geometry.py b/postgresql/types/geometry.py index 000a8845..ba996e57 100644 --- a/postgresql/types/geometry.py +++ b/postgresql/types/geometry.py @@ -114,7 +114,7 @@ class Box(tuple): postgresql.types.geometry.Box(((-2.0, 0.0), (-4.0, -3.0))) :: - + (-2, 0) `high` | | @@ -167,7 +167,7 @@ def __str__(self): class Circle(tuple): """ - type for PostgreSQL circles + Type for PostgreSQL circles. """ __slots__ = () center = property(fget = get0, doc = "center of the circle (point)") From e103bb8ebb7b7133ab9ceea9faffc07ff20a82f8 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Tue, 8 Dec 2020 10:53:12 -0700 Subject: [PATCH 065/109] Relocate the non-english locale section to the bottom as it appears to no longer be relevant. --- postgresql/documentation/gotchas.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/postgresql/documentation/gotchas.rst b/postgresql/documentation/gotchas.rst index beb0a884..915e3360 100644 --- a/postgresql/documentation/gotchas.rst +++ b/postgresql/documentation/gotchas.rst @@ -5,14 +5,6 @@ It is recognized that decisions were made that may not always be ideal for a given user. In order to highlight those potential issues and hopefully bring some sense into a confusing situation, this document was drawn. -Non-English Locales -------------------- - -Many non-english locales are not supported due to the localization of the severity field -in messages and errors sent to the client. Internally, py-postgresql uses this to allow -client side filtering of messages and to identify FATAL connection errors that allow the -client to recognize that it should be expecting the connection to terminate. - Thread Safety ------------- @@ -112,3 +104,11 @@ This exception is raised by a generic processing routine whose functionality is abstract in nature, so the message is abstract as well. It essentially means that a tuple in the sequence given to the loading method had too many or too few items. + +Non-English Locales +------------------- + +In the past, some builds of PostgreSQL localized the severity field of some protocol messages. +`py-postgresql` expects these fields to be consistent with their english terms. If the driver +raises strange exceptions during the use of non-english locales, it may be necessary to use an +english setting in order to coax the server into issueing familiar terms. From a89bd7b396b367f0f9b417e9f8461dddd7373602 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Tue, 8 Dec 2020 11:05:02 -0700 Subject: [PATCH 066/109] Add 1.3 to index. --- postgresql/documentation/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/postgresql/documentation/index.rst b/postgresql/documentation/index.rst index 715b8c08..9189c563 100644 --- a/postgresql/documentation/index.rst +++ b/postgresql/documentation/index.rst @@ -36,6 +36,7 @@ Changes .. toctree:: :maxdepth: 1 + changes-v1.3 changes-v1.2 changes-v1.1 changes-v1.0 From 924e5480556fda97de31acfa1b3e52401b1b0b38 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Wed, 9 Dec 2020 16:17:22 -0700 Subject: [PATCH 067/109] Accept libpq schemes ('postgres://' and 'postgresql://') in addition to pq:// --- postgresql/iri.py | 6 +++--- postgresql/test/test_iri.py | 23 +++++++++++++++++++++-- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/postgresql/iri.py b/postgresql/iri.py index 1ace1ba8..7a90aee1 100644 --- a/postgresql/iri.py +++ b/postgresql/iri.py @@ -14,7 +14,7 @@ Driver Parameters: - pq://user@host/?[driver_param]=value&[other_param]=value?setting=val + pq://user@host/?[driver_param]=value&[other_param]=value?server_setting=val """ from .resolved import riparse as ri from .string import split_ident @@ -30,8 +30,8 @@ def structure(d, fieldproc = ri.unescape): """ Create a clientparams dictionary from a parsed RI. """ - if d.get('scheme', 'pq').lower() != 'pq': - raise ValueError("PQ-IRI scheme is not 'pq'") + if d.get('scheme', 'pq').lower() not in {'pq', 'postgres', 'postgresql'}: + raise ValueError("PQ-IRI scheme is not 'pq', 'postgres', or 'postgresql'") cpd = { k : fieldproc(v) for k, v in d.items() if k not in ('path', 'fragment', 'query', 'host', 'scheme') diff --git a/postgresql/test/test_iri.py b/postgresql/test/test_iri.py index cb6a0abc..0379302f 100644 --- a/postgresql/test/test_iri.py +++ b/postgresql/test/test_iri.py @@ -18,6 +18,7 @@ ':pass@', 'u:p@h', 'u:p@h:1', + 'postgres://host/database', 'pq://user:password@host:port/database?setting=value#public,private', 'pq://fæm.com:123/õéf/á?param=val', 'pq://l»»@fæm.com:123/õéf/á?param=val', @@ -84,6 +85,20 @@ ] class test_iri(unittest.TestCase): + def testAlternateSchemes(self): + field = pg_iri.parse("postgres://host")['host'] + self.assertEqual(field, 'host') + + field = pg_iri.parse("postgresql://host")['host'] + self.assertEqual(field, 'host') + + try: + pg_iri.parse("reject://host") + except ValueError: + pass + else: + self.fail("unacceptable IRI scheme not rejected") + def testIP6Hosts(self): """ Validate that IPv6 hosts are properly extracted. @@ -101,7 +116,9 @@ def testIP6Hosts(self): self.assertEqual(p['host'], h) def testPresentPasswordObscure(self): - "password is present in IRI, and obscure it" + """ + Password is present in IRI, and obscure it. + """ s = 'pq://user:pass@host:port/dbname' o = 'pq://user:***@host:port/dbname' p = pg_iri.parse(s) @@ -109,7 +126,9 @@ def testPresentPasswordObscure(self): self.assertEqual(ps, o) def testPresentPasswordObscure(self): - "password is *not* present in IRI, and do nothing" + """ + Password is *not* present in IRI, and do nothing. + """ s = 'pq://user@host:port/dbname' o = 'pq://user@host:port/dbname' p = pg_iri.parse(s) From 7f3cc3d72398347b024cbf6633c2efcc85166701 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 10 Dec 2020 11:40:24 -0700 Subject: [PATCH 068/109] Use github flavored markdown for the pretty printing. --- README | 67 +++++++++++++++++++++++----------------------------------- 1 file changed, 26 insertions(+), 41 deletions(-) diff --git a/README b/README index bfb53949..3bb196d6 100644 --- a/README +++ b/README @@ -1,5 +1,4 @@ -About -===== +# About py-postgresql is a Python 3 package providing modules for working with PostgreSQL. This includes a high-level driver, and many other tools that support a developer @@ -11,55 +10,41 @@ http://github.com/MagicStack/asyncpg should be considered. py-postgresql, currently, does not have direct support for high-level async interfaces provided by recent versions of Python. Future versions may change this. -Errata ------- +# Errata -.. warning:: - In v1.3, `postgresql.driver.dbapi20.connect` will now raise `ClientCannotConnectError` directly. - Exception traps around connect should still function, but the `__context__` attribute - on the error instance will be `None` in the usual failure case as it is no longer - incorrectly chained. Trapping `ClientCannotConnectError` ahead of `Error` should - allow both cases to co-exist in the event that data is being extracted from - the `ClientCannotConnectError`. +In v1.3, `postgresql.driver.dbapi20.connect` will now raise `ClientCannotConnectError` directly. +Exception traps around connect should still function, but the `__context__` attribute +on the error instance will be `None` in the usual failure case as it is no longer +incorrectly chained. Trapping `ClientCannotConnectError` ahead of `Error` should +allow both cases to co-exist in the event that data is being extracted from +the `ClientCannotConnectError`. -Installation ------------- +# Installation -Installation *should* be as simple as:: +Installation *should* be as simple as: $ python3 ./setup.py install -More information about installation is available via:: +Or: - python -m postgresql.documentation.admin + $ pip install py-postgresql -Basic Driver Usage ------------------- +# Basic Driver Usage -Using PG-API:: +```python + import postgresql + db = postgresql.open('pq://user:password@host:port/database') + get_table = db.prepare("select * from information_schema.tables where table_name = $1") + for x in get_table("tables"): + print(x) + print(get_table.first("tables")) +``` - >>> import postgresql - >>> db = postgresql.open('pq://user:password@host:port/database') - >>> get_table = db.prepare("select * from information_schema.tables where table_name = $1") - >>> for x in get_table("tables"): - >>> print(x) - >>> print(get_table.first("tables")) +# Documentation -However, a DB-API 2.0 driver is provided as well: `postgresql.driver.dbapi20`. +http://py-postgresql.readthedocs.io -Further Information -------------------- +# Related -Online documentation can be retrieved from: - - http://py-postgresql.readthedocs.io - -Or, you can read them in your pager: python -m postgresql.documentation.index - -For information about PostgreSQL: - - http://postgresql.org - -For information about Python: - - http://python.org +- http://postgresql.org +- http://python.org From 4f4007acbd8475aaaf9ab98ffcc499896407eb1a Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 10 Dec 2020 11:41:49 -0700 Subject: [PATCH 069/109] Add type extension. --- README => README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename README => README.md (100%) diff --git a/README b/README.md similarity index 100% rename from README rename to README.md From 8df194bd76fae4836910f21b48014a1840e2ad60 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 10 Dec 2020 11:52:43 -0700 Subject: [PATCH 070/109] Focus the sentence. --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 3bb196d6..3bec06df 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ # About py-postgresql is a Python 3 package providing modules for working with PostgreSQL. -This includes a high-level driver, and many other tools that support a developer -working with PostgreSQL databases. +Primarily, a high-level driver for querying databases. For a high performance async interface, MagicStack's asyncpg http://github.com/MagicStack/asyncpg should be considered. From 2fa49a9885679a70d8412eaf21f87be57157842a Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 10 Dec 2020 11:58:19 -0700 Subject: [PATCH 071/109] Force h3. --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3bec06df..96e4577b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# About +### About py-postgresql is a Python 3 package providing modules for working with PostgreSQL. Primarily, a high-level driver for querying databases. @@ -9,7 +9,7 @@ http://github.com/MagicStack/asyncpg should be considered. py-postgresql, currently, does not have direct support for high-level async interfaces provided by recent versions of Python. Future versions may change this. -# Errata +### Errata In v1.3, `postgresql.driver.dbapi20.connect` will now raise `ClientCannotConnectError` directly. Exception traps around connect should still function, but the `__context__` attribute @@ -18,7 +18,7 @@ incorrectly chained. Trapping `ClientCannotConnectError` ahead of `Error` should allow both cases to co-exist in the event that data is being extracted from the `ClientCannotConnectError`. -# Installation +### Installation Installation *should* be as simple as: @@ -28,7 +28,7 @@ Or: $ pip install py-postgresql -# Basic Driver Usage +### Basic Driver Usage ```python import postgresql @@ -39,11 +39,11 @@ Or: print(get_table.first("tables")) ``` -# Documentation +### Documentation http://py-postgresql.readthedocs.io -# Related +### Related - http://postgresql.org - http://python.org From b40c59b84194aae98ceb4574e86f21b4284f95b6 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 10 Dec 2020 12:00:34 -0700 Subject: [PATCH 072/109] Force line break. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 96e4577b..d534ea83 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ### About -py-postgresql is a Python 3 package providing modules for working with PostgreSQL. +py-postgresql is a Python 3 package providing modules for working with PostgreSQL. Primarily, a high-level driver for querying databases. For a high performance async interface, MagicStack's asyncpg From 8eb65959c8509a3fb869d51781681b6355ecee05 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 10 Dec 2020 12:10:39 -0700 Subject: [PATCH 073/109] Show preference for pypi, but include the clone operation in the manual setup.py call. --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d534ea83..0f397d66 100644 --- a/README.md +++ b/README.md @@ -20,13 +20,15 @@ the `ClientCannotConnectError`. ### Installation -Installation *should* be as simple as: +Using PyPI.org: - $ python3 ./setup.py install + $ pip install py-postgresql -Or: +From a clone: - $ pip install py-postgresql + $ git clone https://github.com/python-postgres/fe.git + $ cd fe + $ python3 ./setup.py install ### Basic Driver Usage From a49a21ac47d27658831e711dc3f4437cd1e10f89 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 10 Dec 2020 12:19:12 -0700 Subject: [PATCH 074/109] Show transaction usage and a streaming Statement.rows instead of first. --- README.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0f397d66..55562d09 100644 --- a/README.md +++ b/README.md @@ -30,15 +30,19 @@ From a clone: $ cd fe $ python3 ./setup.py install -### Basic Driver Usage +### Basic Usage ```python - import postgresql - db = postgresql.open('pq://user:password@host:port/database') - get_table = db.prepare("select * from information_schema.tables where table_name = $1") - for x in get_table("tables"): +import postgresql +db = postgresql.open('pq://user:password@host:port/database') + +get_table = db.prepare("SELECT * from information_schema.tables WHERE table_name = $1") +print(get_table("tables")) + +# Streaming, in a transaction. +with db.xact(): + for x in get_table.rows("tables"): print(x) - print(get_table.first("tables")) ``` ### Documentation From 7f55d92ac265eb8b5f6bc82868089806dd289bbf Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 10 Dec 2020 12:36:52 -0700 Subject: [PATCH 075/109] Add v2.0 warning. --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 55562d09..0d443299 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ http://github.com/MagicStack/asyncpg should be considered. py-postgresql, currently, does not have direct support for high-level async interfaces provided by recent versions of Python. Future versions may change this. -### Errata +### Advisory In v1.3, `postgresql.driver.dbapi20.connect` will now raise `ClientCannotConnectError` directly. Exception traps around connect should still function, but the `__context__` attribute @@ -18,6 +18,9 @@ incorrectly chained. Trapping `ClientCannotConnectError` ahead of `Error` should allow both cases to co-exist in the event that data is being extracted from the `ClientCannotConnectError`. +In v2.0, support for older versions of PostgreSQL and Python will be removed. +If you have automated installations using PyPI, make sure that they specify a major version. + ### Installation Using PyPI.org: From 2fa852d6ef8e49e63062a7f7f1d122319a0f33b8 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 10 Dec 2020 13:43:08 -0700 Subject: [PATCH 076/109] Note that the driver can be used as-is. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0d443299..e75f8722 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ From a clone: $ git clone https://github.com/python-postgres/fe.git $ cd fe - $ python3 ./setup.py install + $ python3 ./setup.py install # Or use in-place without installation(PYTHONPATH). ### Basic Usage From 6731fe3be5f6a2cabc6b42fd006f92f56f729762 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sat, 19 Dec 2020 12:01:18 -0700 Subject: [PATCH 077/109] Add Connection.transaction alias for asyncpg consistency. --- postgresql/documentation/changes-v1.3.rst | 1 + .../documentation/sphinx/changes-v1.3.rst | 1 + postgresql/driver/pq3.py | 5 ++++- postgresql/test/test_driver.py | 21 +++++++++++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) create mode 120000 postgresql/documentation/sphinx/changes-v1.3.rst diff --git a/postgresql/documentation/changes-v1.3.rst b/postgresql/documentation/changes-v1.3.rst index 5ec51561..8b8686c3 100644 --- a/postgresql/documentation/changes-v1.3.rst +++ b/postgresql/documentation/changes-v1.3.rst @@ -6,6 +6,7 @@ Changes in v1.3 * Commit DB-API 2.0 ClientCannotConnect exception correction. * Eliminate types-as-documentation annotations. + * Add Connection.transaction alias for asyncpg consistency. * Eliminate multiple inheritance in `postgresql.api` in favor of ABC registration. * Add support for PGTEST environment variable (pq-IRI) to improve test performance and to aid in cases where the target fixture is already available. diff --git a/postgresql/documentation/sphinx/changes-v1.3.rst b/postgresql/documentation/sphinx/changes-v1.3.rst new file mode 120000 index 00000000..51c4b25d --- /dev/null +++ b/postgresql/documentation/sphinx/changes-v1.3.rst @@ -0,0 +1 @@ +../changes-v1.3.rst \ No newline at end of file diff --git a/postgresql/driver/pq3.py b/postgresql/driver/pq3.py index a42d0bab..24976de0 100644 --- a/postgresql/driver/pq3.py +++ b/postgresql/driver/pq3.py @@ -2324,9 +2324,12 @@ def do(self, language : str, source : str, sql = "DO " + qlit(source) + " LANGUAGE " + qid(language) + ";" self.execute(sql) - def xact(self, isolation = None, mode = None) -> Transaction: + # Alias transaction as xact. xact is the original term, but support + # the full word for identifier consistency with asyncpg. + def transaction(self, isolation = None, mode = None) -> Transaction: x = Transaction(self, isolation = isolation, mode = mode) return x + xact=transaction def prepare(self, sql_statement_string : str, diff --git a/postgresql/test/test_driver.py b/postgresql/test/test_driver.py index f590bf12..df413314 100644 --- a/postgresql/test/test_driver.py +++ b/postgresql/test/test_driver.py @@ -824,6 +824,27 @@ def testSelectInXact(self): with db.xact(): self.select() + @pg_tmp + def testTransactionAlias(self): + self.assertEqual(db.transaction, db.xact) + + try: + with db.transaction(): + db.execute("CREATE TABLE t (i int);") + raise Exception('some failure') + except: + pass + else: + self.fail("expected exception was not raised") + + try: + db.query("select * from t") + except: + # No table. + pass + else: + self.fail("transaction abort had no effect") + def cursor_read(self): ps = db.prepare("SELECT i FROM generate_series(0, (2^8)::int - 1) AS g(i)") c = ps.declare() From 8775f707aed24359a724a9e28e03699cd7aecf87 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sat, 19 Dec 2020 12:15:28 -0700 Subject: [PATCH 078/109] Add editorconfig and remove vim configuration lines. --- .editorconfig | 6 ++++++ postgresql/api.py | 2 -- postgresql/bin/pg_python.py | 2 -- postgresql/cluster.py | 2 -- postgresql/configfile.py | 2 -- postgresql/exceptions.py | 2 -- postgresql/port/_optimized/buffer.c | 3 --- postgresql/port/_optimized/functools.c | 3 --- postgresql/port/_optimized/module.c | 3 --- postgresql/port/_optimized/wirestate.c | 3 --- postgresql/python/command.py | 2 -- 11 files changed, 6 insertions(+), 24 deletions(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..988c986a --- /dev/null +++ b/.editorconfig @@ -0,0 +1,6 @@ +root = true + +[*.{py,c}] +indent_style = tab +indent_size = tab +tab_width = 4 diff --git a/postgresql/api.py b/postgresql/api.py index 4480bd40..e177c30e 100644 --- a/postgresql/api.py +++ b/postgresql/api.py @@ -1377,5 +1377,3 @@ def __exit__(self, exc, val, tb): __docformat__ = 'reStructuredText' if __name__ == '__main__': help(__package__ + '.api') -## -# vim: ts=3:sw=3:noet: diff --git a/postgresql/bin/pg_python.py b/postgresql/bin/pg_python.py index 28d993ec..a97aa97b 100644 --- a/postgresql/bin/pg_python.py +++ b/postgresql/bin/pg_python.py @@ -134,5 +134,3 @@ def command(argv = sys.argv): if __name__ == '__main__': sys.exit(command(sys.argv)) -## -# vim: ts=3:sw=3:noet: diff --git a/postgresql/cluster.py b/postgresql/cluster.py index bb0ff0d9..122103eb 100644 --- a/postgresql/cluster.py +++ b/postgresql/cluster.py @@ -635,5 +635,3 @@ def wait_until_stopped(self, timeout = 10, delay = 0.05): creator = self, ) time.sleep(delay) -## -# vim: ts=3:sw=3:noet: diff --git a/postgresql/configfile.py b/postgresql/configfile.py index 18d312e8..2f94f0e1 100644 --- a/postgresql/configfile.py +++ b/postgresql/configfile.py @@ -317,5 +317,3 @@ def getset(self, keys): for x in (keys - set(cfg.keys())): cfg[x] = None return cfg -## -# vim: ts=3:sw=3:noet: diff --git a/postgresql/exceptions.py b/postgresql/exceptions.py index 39cf5a33..5d71d01c 100644 --- a/postgresql/exceptions.py +++ b/postgresql/exceptions.py @@ -748,5 +748,3 @@ def code_lookup( ) ) ) -## -# vim: ts=3:sw=3:noet: diff --git a/postgresql/port/_optimized/buffer.c b/postgresql/port/_optimized/buffer.c index 0b6cf2eb..7f46d2d1 100644 --- a/postgresql/port/_optimized/buffer.c +++ b/postgresql/port/_optimized/buffer.c @@ -624,6 +624,3 @@ PyTypeObject pq_message_stream_Type = { p_new, /* tp_new */ NULL, /* tp_free */ }; -/* - * vim: ts=3:sw=3:noet: - */ diff --git a/postgresql/port/_optimized/functools.c b/postgresql/port/_optimized/functools.c index 45a5d75f..9a0deea0 100644 --- a/postgresql/port/_optimized/functools.c +++ b/postgresql/port/_optimized/functools.c @@ -335,6 +335,3 @@ compose(PyObject *self, PyObject *args) return(rob); } -/* - * vim: ts=3:sw=3:noet: - */ diff --git a/postgresql/port/_optimized/module.c b/postgresql/port/_optimized/module.c index 240921d5..33f68759 100644 --- a/postgresql/port/_optimized/module.c +++ b/postgresql/port/_optimized/module.c @@ -149,6 +149,3 @@ PyInit_optimized(void) Py_DECREF(mod); return(NULL); } -/* - * vim: ts=3:sw=3:noet: - */ diff --git a/postgresql/port/_optimized/wirestate.c b/postgresql/port/_optimized/wirestate.c index 9af150cb..7947e5cb 100644 --- a/postgresql/port/_optimized/wirestate.c +++ b/postgresql/port/_optimized/wirestate.c @@ -284,6 +284,3 @@ PyTypeObject WireState_Type = { ws_new, /* tp_new */ NULL, /* tp_free */ }; -/* - * vim: ts=3:sw=3:noet: - */ diff --git a/postgresql/python/command.py b/postgresql/python/command.py index 35fa8ab7..18685c22 100644 --- a/postgresql/python/command.py +++ b/postgresql/python/command.py @@ -633,5 +633,3 @@ def command(argv = sys.argv): if __name__ == '__main__': sys.exit(command()) -## -# vim: ts=3:sw=3:noet: From 8ddf5a4702eafb97e12d76dbdf952aaff7a26ecd Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Thu, 18 Nov 2021 23:13:30 -0400 Subject: [PATCH 079/109] Python 3.10: Update another collections.abc import --- postgresql/copyman.py | 2 +- postgresql/documentation/notifyman.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/postgresql/copyman.py b/postgresql/copyman.py index b0f7a6c0..43229822 100644 --- a/postgresql/copyman.py +++ b/postgresql/copyman.py @@ -9,7 +9,7 @@ """ import sys from abc import abstractmethod, abstractproperty -from collections import Iterator +from collections.abc import Iterator from .python.element import Element, ElementSet from .python.structlib import ulong_unpack, ulong_pack from .protocol.buffer import pq_message_stream diff --git a/postgresql/documentation/notifyman.rst b/postgresql/documentation/notifyman.rst index d774ee52..0b214750 100644 --- a/postgresql/documentation/notifyman.rst +++ b/postgresql/documentation/notifyman.rst @@ -20,7 +20,7 @@ receives notifications. The `postgresql.notifyman.NotificationManager` class is used to wait for messages to come in on a set of connections, pick up the messages, and deliver -the messages to the object's user via the `collections.Iterator` protocol. +the messages to the object's user via the `collections.abc.Iterator` protocol. Listening on a Single Connection From 21ecf9b517e8e8b9053bc741c1a161c4d91a544b Mon Sep 17 00:00:00 2001 From: James William Pye Date: Wed, 21 Sep 2022 22:25:09 -0700 Subject: [PATCH 080/109] Formatting --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e75f8722..a987adf4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ### About -py-postgresql is a Python 3 package providing modules for working with PostgreSQL. +py-postgresql is a Python 3 package providing modules for working with PostgreSQL. Primarily, a high-level driver for querying databases. For a high performance async interface, MagicStack's asyncpg @@ -18,7 +18,7 @@ incorrectly chained. Trapping `ClientCannotConnectError` ahead of `Error` should allow both cases to co-exist in the event that data is being extracted from the `ClientCannotConnectError`. -In v2.0, support for older versions of PostgreSQL and Python will be removed. +In v2.0, support for older versions of PostgreSQL and Python will be removed. If you have automated installations using PyPI, make sure that they specify a major version. ### Installation From 421ef714711ceda42877b1e0bd171ffc88ad911d Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 23 Sep 2022 13:33:56 -0700 Subject: [PATCH 081/109] Update Python requirement. --- postgresql/release/distutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/release/distutils.py b/postgresql/release/distutils.py index 1d414488..bf195296 100644 --- a/postgresql/release/distutils.py +++ b/postgresql/release/distutils.py @@ -177,7 +177,7 @@ def standard_setup_keywords(build_extensions = True, prefix = default_prefix): 'packages' : list(prefixed_packages(prefix = prefix)), 'package_data' : dict(prefixed_package_data(prefix = prefix)), 'cmdclass': dict(test=TestCommand), - 'python_requires': '>=3.3', + 'python_requires': '>=3.8', } if build_extensions: d['ext_modules'] = list(prefixed_extensions(prefix = prefix)) From f768eeb6b7a7b564b876e425f87e139f2796e7b4 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Wed, 16 Nov 2022 13:00:07 -0700 Subject: [PATCH 082/109] s/failUnlessEqual/assertEqual/g --- postgresql/test/cursor_integrity.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/postgresql/test/cursor_integrity.py b/postgresql/test/cursor_integrity.py index 14a07acf..7bad07f2 100644 --- a/postgresql/test/cursor_integrity.py +++ b/postgresql/test/cursor_integrity.py @@ -55,7 +55,7 @@ def test_select(self): read += thisread completed.append(next[0]) if thisread: - self.failUnlessEqual( + self.assertEqual( last[0][-1][0], next[0][0][0] - 1, "first row(-1) of next failed to match the last row of the previous" ) @@ -63,8 +63,8 @@ def test_select(self): elif next[1] != 0: # done break - self.failUnlessEqual(read, limit) - self.failUnlessEqual(list(range(-1, limit)), [ + self.assertEqual(read, limit) + self.assertEqual(list(range(-1, limit)), [ x[0] for x in itertools.chain(*completed) ]) @@ -88,7 +88,7 @@ def test_copy_out(self): read += thisread completed.append(next[0]) if thisread: - self.failUnlessEqual( + self.assertEqual( last[0][-1], next[0][0] - 1, "first row(-1) of next failed to match the last row of the previous" ) @@ -96,8 +96,8 @@ def test_copy_out(self): elif next[1] != 0: # done break - self.failUnlessEqual(read, limit) - self.failUnlessEqual( + self.assertEqual(read, limit) + self.assertEqual( list(range(-1, limit)), list(itertools.chain(*completed)) ) From dc07e2d86feedd84ce3cdb5adbae96c4f2866113 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sat, 4 Feb 2023 11:14:34 -0700 Subject: [PATCH 083/109] Add pyproject.toml file. --- pyproject.toml | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..2ff7b1fa --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 -m validate_pyproject +# Consistent with postgresql.project and postgresql.release.distutils. +[build-system] +requires = ["setuptools >= 0"] +build-backend = "setuptools.build_meta" + +[project] +name = "py-postgresql" +version = "1.3.1" +description = "Query PostgreSQL databases using Python and the PQv3 protocol." +readme = "README.md" + +license.file = "LICENSE" +authors = [ + { name = "James William Pye", email = "james.pye@gmail.com" }, +] + +requires-python = ">=3.7" +keywords = ["syncpg", "postgres", "postgresql", "sql", "driver"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "License :: OSI Approved :: MIT License", + "License :: OSI Approved :: Attribution Assurance License", + "License :: OSI Approved :: Python Software Foundation License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Topic :: Database", +] + +dependencies = [] + +[project.urls] +Documentation = "http://py-postgresql.readthedocs.io" +Issues = "https://github.com/python-postgres/fe/issues" +Source = "https://github.com/python-postgres/fe" From 140efeedb857a206c81b2e97fe884a8f57f5a3a4 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sat, 4 Feb 2023 11:16:07 -0700 Subject: [PATCH 084/109] Correct version requirement in release.distutils and github project identifier in postgresql.project. --- postgresql/project.py | 2 +- postgresql/release/distutils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/postgresql/project.py b/postgresql/project.py index c4d8516a..85e77bdf 100644 --- a/postgresql/project.py +++ b/postgresql/project.py @@ -5,7 +5,7 @@ name = 'py-postgresql' identity = 'http://github.com/python-postgres/fe' -meaculpa = 'Python+Postgres' +meaculpa = 'python-postgres' abstract = 'Driver and tools library for PostgreSQL' version_info = (1, 3, 0) diff --git a/postgresql/release/distutils.py b/postgresql/release/distutils.py index bf195296..a18af4f3 100644 --- a/postgresql/release/distutils.py +++ b/postgresql/release/distutils.py @@ -177,7 +177,7 @@ def standard_setup_keywords(build_extensions = True, prefix = default_prefix): 'packages' : list(prefixed_packages(prefix = prefix)), 'package_data' : dict(prefixed_package_data(prefix = prefix)), 'cmdclass': dict(test=TestCommand), - 'python_requires': '>=3.8', + 'python_requires': '>=3.7', } if build_extensions: d['ext_modules'] = list(prefixed_extensions(prefix = prefix)) From de37467e8e09c1fc77c3c6d6b4063a027b2c0e40 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sat, 4 Feb 2023 11:16:21 -0700 Subject: [PATCH 085/109] Update patch level. --- postgresql/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/project.py b/postgresql/project.py index 85e77bdf..34ddf3a6 100644 --- a/postgresql/project.py +++ b/postgresql/project.py @@ -8,5 +8,5 @@ meaculpa = 'python-postgres' abstract = 'Driver and tools library for PostgreSQL' -version_info = (1, 3, 0) +version_info = (1, 3, 1) version = '.'.join(map(str, version_info)) From 63bdd6085432e46d43274aaf4f79738325853900 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sat, 4 Feb 2023 11:35:40 -0700 Subject: [PATCH 086/109] Note maintainers in pyproject.toml. --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 2ff7b1fa..f581b945 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,9 @@ license.file = "LICENSE" authors = [ { name = "James William Pye", email = "james.pye@gmail.com" }, ] +maintainers = [ + { name = "James William Pye", email = "james.pye@gmail.com" }, +] requires-python = ">=3.7" keywords = ["syncpg", "postgres", "postgresql", "sql", "driver"] From a91574c1f1abb107163c3260cddfce4c86d8af2d Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sat, 4 Feb 2023 19:38:52 -0700 Subject: [PATCH 087/109] Recognize '--with-ssl=' configure option used in newer version of PostgreSQL. --- postgresql/installation.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/postgresql/installation.py b/postgresql/installation.py index 046bff44..5856816c 100644 --- a/postgresql/installation.py +++ b/postgresql/installation.py @@ -240,7 +240,13 @@ def ssl(self): """ Whether the installation was compiled with SSL support. """ - return 'with_openssl' in self.configure_options + if 'with_openssl' in self.configure_options: + return True + # Parameterized form in newer versions. + for x in self.configure_options: + if 'with_ssl' in x: + return True + return False def default(typ = Installation): """ From 2bb3852b8fb4ec8e3ec69d66dce7590c4dbeeed0 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sun, 5 Feb 2023 09:52:38 -0700 Subject: [PATCH 088/109] Accept arbitrary keywords with api.Connector and normalize doc-string. --- postgresql/api.py | 1 + postgresql/driver/__init__.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/postgresql/api.py b/postgresql/api.py index e177c30e..858c3f2b 100644 --- a/postgresql/api.py +++ b/postgresql/api.py @@ -1129,6 +1129,7 @@ def __init__(self, database : str = None, settings : (dict, [(str,str)]) = None, category : Category = None, + **kw, ): if user is None: # sure, it's a "required" keyword, makes for better documentation diff --git a/postgresql/driver/__init__.py b/postgresql/driver/__init__.py index c7c2c433..93f014fd 100644 --- a/postgresql/driver/__init__.py +++ b/postgresql/driver/__init__.py @@ -10,5 +10,7 @@ default = Driver() def connect(*args, **kw): - 'Establish a connection using the default driver.' + """ + Establish a connection using the default driver. + """ return default.connect(*args, **kw) From 188a2436a325b3b8e328aa1e16d50ba30681ed31 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sun, 5 Feb 2023 09:58:30 -0700 Subject: [PATCH 089/109] Avoid abstractmethods on non-ABCMeta instances. --- postgresql/driver/pq3.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/postgresql/driver/pq3.py b/postgresql/driver/pq3.py index 24976de0..f0b7ef3e 100644 --- a/postgresql/driver/pq3.py +++ b/postgresql/driver/pq3.py @@ -10,7 +10,6 @@ from traceback import format_exception from itertools import repeat, chain, count from functools import partial -from abc import abstractmethod from codecs import lookup as lookup_codecs from operator import itemgetter @@ -631,12 +630,12 @@ class Output(object): _complete_message = None - @abstractmethod def _init(self): """ Bind a cursor based on the configured parameters. """ # The local initialization for the specific cursor. + raise NotImplementedError def __init__(self, cursor_id, wref = weakref.ref, ID = ID): self.cursor_id = cursor_id @@ -990,17 +989,17 @@ def __init__(self, statement, parameters, cursor_id): self.database = statement.database Output.__init__(self, cursor_id or ID(self)) - @abstractmethod def _bind(self): """ Generate the commands needed to bind the cursor. """ + raise NotImplementedError - @abstractmethod def _fetch(self): """ Generate the commands needed to bind the cursor. """ + raise NotImplementedError def _init(self): self._command = self._fetch() @@ -2768,7 +2767,6 @@ def __repr__(self): keywords = os.linesep + ' ' + keywords if keywords else '' ) - @abstractmethod def socket_factory_sequence(self): """ Generate a list of callables that will be used to attempt to make the @@ -2778,6 +2776,7 @@ def socket_factory_sequence(self): The callables in the sequence must take a timeout parameter. """ + raise NotImplementedError def __init__(self, connect_timeout : int = None, @@ -2843,23 +2842,12 @@ def __init__(self, for k, v in tnkw.items() ]) self._password = (self.password or '').encode(se) - self._socket_secure = { - 'keyfile' : self.sslkeyfile, - 'certfile' : self.sslcrtfile, - 'ca_certs' : self.sslrootcrtfile, - } # class Connector class SocketConnector(Connector): """ Abstract connector for using `socket` and `ssl`. """ - @abstractmethod - def socket_factory_sequence(self): - """ - Return a sequence of `SocketFactory`s for a connection to use to connect - to the target host. - """ def create_socket_factory(self, **params): return SocketFactory(**params) From 9386fd5d321dbc2542f2b8b629e1361a86c4877b Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sun, 5 Feb 2023 13:45:05 -0700 Subject: [PATCH 090/109] Update self signed certificate to something recent Pythons accepts. --- postgresql/test/test_ssl_connect.py | 159 +++++++++++++++------------- 1 file changed, 83 insertions(+), 76 deletions(-) diff --git a/postgresql/test/test_ssl_connect.py b/postgresql/test/test_ssl_connect.py index fc72c86c..f9b90a6d 100644 --- a/postgresql/test/test_ssl_connect.py +++ b/postgresql/test/test_ssl_connect.py @@ -18,87 +18,94 @@ server_key = """ -----BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQCy8veVaqL6MZVT8o0j98ggZYfibGwSN4XGC4rfineA2QZhi8t+ -zrzfOS10vLXKtgiIpevHeQbDlrqFDPUDowozurg+jfro2L1jzQjZPdgqOUs+YjKh -EO0Ya7NORO7ZgBx8WveXq30k4l8DK41jvpxRyBb9aqNWG4cB7fJqVTwZrwIDAQAB -AoGAJ74URGfheEVoz7MPq4xNMvy5mAzSV51jJV/M4OakscYBR8q/UBNkGQNe2A1N -Jo8VCBwpaCy11txz4jbFd6BPFFykgXleuRvMxoTv1qV0dZZ0X0ESNEAnjoHtjin/ -25mxsZTR6ucejHqXD9qE9NvFQ+wLv6Xo5rgDpx0onvgLA3kCQQDn4GeMkCfPZCve -lDUK+TpJnLYupyElZiidoFMITlFo5WoWNJror2W42A5TD9sZ23pGSxw7ypiWIF4f -ukGT5ZSzAkEAxZDwUUhgtoJIK7E9sCJM4AvcjDxGjslbUI/SmQTT+aTNCAmcIRrl -kq3WMkPjxi/QFEdkIpPsV9Kc94oQ/8b9FQJBAKHxRQCTsWoTsNvbsIwAcif1Lfu5 -N9oR1i34SeVUJWFYUFY/2SzHSwjkxGRYf5I4idZMIOTVYun+ox4PjDtJrScCQEQ4 -RiNrIKok1pLvwuNdFLqQnfl2ns6TTQrGfuwDtMaRV5Mc7mKoDPnXOQ1mT/KRdAJs -nHEsLwIsYbNAY5pOtfkCQDOy2Ffe7Z1YzFZXCTzpcq4mvMOPEUqlIX6hACNJGhgt -1EpruPwqR2PYDOIC4sXCaSogL8YyjI+Jlhm5kEJ4GaU= +MIIJKQIBAAKCAgEAp6C6t3exwgx5QQjeoW2vtawSl9SMhsNKfwGVh97gStBCHNqZ +DuO6nn5qp3GmzkDII+B8uAJPe5znHSlqj2g13EiFENeaF3G9l1uzaWGEvuFyU2sq +x3lu/pJz6ISEhlogkrGz9inmMcLaNLzm4XbXR/9pjf3QKq7xPH0CacjSzeA9gfAm +CjKJM/DxkrWeyKvBJuVCZbDPbCHtS1MJvAcU0DL9wdfPr4+2P4rVjzBgbzUzPUXL +DT/ewAk94aJPZAWAvtNrdbXjSvIJ/CWBedLtyCpHPchRwaOdJrkZYItYRqYP7SKM +rwddTbrQ/70sCHCS9Yq7X6NO9ONNVrgLhVQm3Ua3FsGyKcU/bx+xEYQsAsCJj7Ps +WdRhImU/3bdHqPobwyKRbssa5iz1rrwdQ0eFUakv+he3nqXLUqmqOs4RrrM5OvRs +e/JCi5N50NlRXkiix2u909vCdPQFzviiVQbkpqzSmejN0PF3GYFpc0+c7HDp78J9 +YEd+WMvx16LABVy9Kq5eYQbGQOmaWzH01fH+h3vxGnA4G9ArXGPL93P0+ztNhHJf +XBg5bwNzy5cca4roy6QNx87M/+n23iEHE4Bn9uulYJsXx2urUOAN9WCJTKYULTfu +IChWIRDy5ceYVcedHiuhRO90WyGsmwILAoAV0jebDosMK6Q+kIoOM2DT1n8CAwEA +AQKCAgAoDKTPtM9Jl4VY3m+ijfxPIX+HuwagJASmd5BsV/mqpjtFfYzYG9y4hWeh +/etml9+5gqcJp7OpywEE3KJTBQjpSoJQVdLBCzHK+ePRp7T5jg+skow0AHVeaUs8 +IH0xRFNH+SEQDU6sUOulcgSPlb81unZTsHKN4CJO22c6Mvr6qTrI0sGj6hMRz91H +uhDnzPFnA5trhGTqZui0+G/49pAodiZeq9s5DNL0N41ympJPv5wwZX5v+fSUWSDp +ycfCE/aAoS6pfv2BKHbuQV+/5X9eNYuz3Sp7Y0XmvI6tnF1I8+AWPgzyvIW0TpAk +qePdWFgkRjMiVHhG1g/iSjKmdkaacIqfOmaaUO//r5uj8L4rSxcTtqfM4CoUwiGo +Iqj0fCMQp+G+QCyMJzm0d2Ctg5mOMxFbl3jk09Z2u/ZaUpauvJ0S8WIEkJ0BMdGN +AqOtBFD7xOo6od2+7zreBVJQAV15owwi578Jk86skp5zY400IlV51yLqM6BQa3zd +Ft4xF2up0e/n7xVWW3twY8m/4i+ie+20hap9730UENo1XGy9iIIN8bxHU9+NcHdL +AP71Zgqa2nC2Wy7sCdRkt8c7P2VuraOoWgsOFvShhNOdVWR/LH/WwQaBzG9nj2u3 +0hmaDJezjdBGEJk3EhNocrMxYV1+L/MBgT6jBABx0b98K9YAEQKCAQEA03u/PPBQ +9H3ybGHC/JpfG+hLd6/Ux8Eq69i4rrduYshRFnwKtSjyfAPdtpLPhx3/N3uuxmCK +2VO+hwMKCEgk8Sb/qp0z2Dthvl2KCHUXOilFEh6B5J1nQi3OJVxalR3/yDp3ir8F +TdptY1gybWBFGdwnKSdHEiCr3+3k9OoSPeqYUuHuzBw2s8Zcet8RirJbtJVv5VyY +WNnzv/vDaEsMLENm0opmEAkFw+YW3ltolgPXyKfmgtXOKAk2s04DEwC0sHmiUvBY +4CdX6TBD7DXNEkl+bpOA5j92USGrlAiKkxeq+igQ7dPDhlmYcAahJodV3fyBPwqz +5pa6SWxQMNhRYwKCAQEAyum6pSHGHoI8twxpf/sgG3wKwUN+BQXoKustueeU66GV +P5xN+4tFmxJCRegFnfRB/IS9Oi5tety1BgYUA8z7h3pRz3ed3FUF0UXCDhgnqmp3 +XWpa9MBkoA8MO+/s+k10CZz5doR9cS+l2c7XfrlwHn21juScfGEsaxhgGBYbDVlW +IehjNERjVYyl4oHG9H/baXGRLaYfaFummwNGivWI0kqn8b00Sc0uW28LAmze5/IM +2simidgDJjV8EScta8o8uF6fe/3WKvGas7/NwVW+zP+Rs/sgsqa9FHQ2FZYRzIrw +5VpnGbz69SxRkbqLdPoKNQrcGOUdDmXrNZds1BAfNQKCAQEArztnHzhE7AD8ASAU +L7g9vGMDPT3dQlLlnJxrkqF8/q7auZW4TZmLKoUNjf0hpeSOF0wNamSOSDtisH4t +LuWQbp0Q1S8CyVWSzOi2ugFDaLbPe475tBNUfvpzSHO4vrwnt6HycW2MGJE3eEyZ +JBXTy/SmIixgcD3QDHES+HiG+vTKmEqK0mdCUD25XTo+T70vzXbRS6wos96MYPRc +Wqtsf7StmyCAJyNCuqqJIl99TmgKwUGV96zu8C+KOpIWbAV2so9ml/B8w+b1qcuL +TErcDB4He9oOwTmucNVEVRmqsOy4iCTwug9wgH72l0R2/PTAinpyIWld3V/hJXtx +CrgC3wKCAQEAiTTQk3ap+9k+6tvGvtZ1WIBg2Vwk64qZ+eN60PlKFqb1P8UWaiA7 +mecXzyNcIPmYYQL03VGlj+2Lrp4PjJ5f+rT4etw8b09ClsafuF4W/EHvosgW5ubt +Y9mpASJ0ULBs5U8y1DQ0ioOYlxYpWzRTHxsL2Kq3MdeXbHdYCxFvi3A8MMNtyVrw +/FkVlnsAqDWIjN1RONfa5vsKRklJuw7aTLBUrb6ti7XlQchtXl91vstKa+o/yne5 +cW27DfI64Wcn9ddt6i6zUeh7Hk509+VeFko+IMCP1J2wvxLxu1j1giT1TXD6xEmo +PH6STYMhZ6DnpARK3b6XDjRWfq981ExufQKCAQAwPtgINZF5c5GjIyn0EZh5cK4l +Ef7E7qXHFYV9yH4dswE9hdOD6IggaZTv1XvrwH5SN9Kt8FOXbPMsARSlD1kUNWsl +aTuco3xmCQNtq/ydN3OGMKOgUV3egzc1xWKSB8txOnfwZyEoHyCT6EQUQgF0ePLm +jcq9ONsyyLWZnRc7qxfJIwb7zCNAvOQezd+J+sDcqUQShfc3tzhqLmfaEOQz/Bz/ +4Sy6OIsujW1LWiJ//B0QXxxhjWd8NKmuTQC1cyKKXUh8iXvAO0CNjhdcZxjN+07n +JSuuwpLFnQtfda1VpNg0seYqbihuuVJpOA55/tlu1BiakdIW6DHB0wrMOL50 -----END RSA PRIVATE KEY----- """ server_crt = """ -Certificate: - Data: - Version: 3 (0x2) - Serial Number: - a1:02:62:34:22:0d:45:6a - Signature Algorithm: md5WithRSAEncryption - Issuer: C=US, ST=Arizona, L=Nowhere, O=ACME Inc, OU=Test Division, CN=test.python.projects.postgresql.org - Validity - Not Before: Feb 18 15:52:20 2009 GMT - Not After : Mar 20 15:52:20 2009 GMT - Subject: C=US, ST=Arizona, L=Nowhere, O=ACME Inc, OU=Test Division, CN=test.python.projects.postgresql.org - Subject Public Key Info: - Public Key Algorithm: rsaEncryption - RSA Public Key: (1024 bit) - Modulus (1024 bit): - 00:b2:f2:f7:95:6a:a2:fa:31:95:53:f2:8d:23:f7: - c8:20:65:87:e2:6c:6c:12:37:85:c6:0b:8a:df:8a: - 77:80:d9:06:61:8b:cb:7e:ce:bc:df:39:2d:74:bc: - b5:ca:b6:08:88:a5:eb:c7:79:06:c3:96:ba:85:0c: - f5:03:a3:0a:33:ba:b8:3e:8d:fa:e8:d8:bd:63:cd: - 08:d9:3d:d8:2a:39:4b:3e:62:32:a1:10:ed:18:6b: - b3:4e:44:ee:d9:80:1c:7c:5a:f7:97:ab:7d:24:e2: - 5f:03:2b:8d:63:be:9c:51:c8:16:fd:6a:a3:56:1b: - 87:01:ed:f2:6a:55:3c:19:af - Exponent: 65537 (0x10001) - X509v3 extensions: - X509v3 Subject Key Identifier: - 4B:2F:4F:1A:43:75:43:DC:26:59:89:48:56:73:BB:D0:AA:95:E8:60 - X509v3 Authority Key Identifier: - keyid:4B:2F:4F:1A:43:75:43:DC:26:59:89:48:56:73:BB:D0:AA:95:E8:60 - DirName:/C=US/ST=Arizona/L=Nowhere/O=ACME Inc/OU=Test Division/CN=test.python.projects.postgresql.org - serial:A1:02:62:34:22:0D:45:6A - - X509v3 Basic Constraints: - CA:TRUE - Signature Algorithm: md5WithRSAEncryption - 24:ee:20:0f:b5:86:08:d6:3c:8f:d4:8d:16:fd:ac:e8:49:77: - 86:74:7d:b8:f3:15:51:1d:d8:65:17:5e:a8:58:aa:b0:f6:68: - 45:cb:77:9d:9f:21:81:e3:5e:86:1c:64:31:39:b6:29:5f:f1: - ec:b1:33:45:1f:0c:54:16:26:11:af:e2:23:1b:a6:03:46:9b: - 0e:63:ce:2c:02:41:26:93:bc:6f:6e:08:7e:95:b7:7a:f9:3a: - 5a:bd:47:4c:92:ce:ea:09:75:de:3d:bb:30:51:a0:c5:f1:5d: - 33:5f:c0:37:75:53:4e:6c:b4:3b:b1:a5:1b:fd:59:19:07:18: - 22:6a -----BEGIN CERTIFICATE----- -MIIDhzCCAvCgAwIBAgIJAKECYjQiDUVqMA0GCSqGSIb3DQEBBAUAMIGKMQswCQYD -VQQGEwJVUzEQMA4GA1UECBMHQXJpem9uYTEQMA4GA1UEBxMHTm93aGVyZTERMA8G -A1UEChMIQUNNRSBJbmMxFjAUBgNVBAsTDVRlc3QgRGl2aXNpb24xLDAqBgNVBAMT -I3Rlc3QucHl0aG9uLnByb2plY3RzLnBvc3RncmVzcWwub3JnMB4XDTA5MDIxODE1 -NTIyMFoXDTA5MDMyMDE1NTIyMFowgYoxCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdB -cml6b25hMRAwDgYDVQQHEwdOb3doZXJlMREwDwYDVQQKEwhBQ01FIEluYzEWMBQG -A1UECxMNVGVzdCBEaXZpc2lvbjEsMCoGA1UEAxMjdGVzdC5weXRob24ucHJvamVj -dHMucG9zdGdyZXNxbC5vcmcwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBALLy -95VqovoxlVPyjSP3yCBlh+JsbBI3hcYLit+Kd4DZBmGLy37OvN85LXS8tcq2CIil -68d5BsOWuoUM9QOjCjO6uD6N+ujYvWPNCNk92Co5Sz5iMqEQ7Rhrs05E7tmAHHxa -95erfSTiXwMrjWO+nFHIFv1qo1YbhwHt8mpVPBmvAgMBAAGjgfIwge8wHQYDVR0O -BBYEFEsvTxpDdUPcJlmJSFZzu9CqlehgMIG/BgNVHSMEgbcwgbSAFEsvTxpDdUPc -JlmJSFZzu9CqlehgoYGQpIGNMIGKMQswCQYDVQQGEwJVUzEQMA4GA1UECBMHQXJp -em9uYTEQMA4GA1UEBxMHTm93aGVyZTERMA8GA1UEChMIQUNNRSBJbmMxFjAUBgNV -BAsTDVRlc3QgRGl2aXNpb24xLDAqBgNVBAMTI3Rlc3QucHl0aG9uLnByb2plY3Rz -LnBvc3RncmVzcWwub3JnggkAoQJiNCINRWowDAYDVR0TBAUwAwEB/zANBgkqhkiG -9w0BAQQFAAOBgQAk7iAPtYYI1jyP1I0W/azoSXeGdH248xVRHdhlF16oWKqw9mhF -y3ednyGB416GHGQxObYpX/HssTNFHwxUFiYRr+IjG6YDRpsOY84sAkEmk7xvbgh+ -lbd6+TpavUdMks7qCXXePbswUaDF8V0zX8A3dVNObLQ7saUb/VkZBxgiag== +MIIGOjCCBCKgAwIBAgIUPZomw8k4yyMSlXYFUrsQ7Co8LCowDQYJKoZIhvcNAQEL +BQAwgawxCzAJBgNVBAYTAlVTMRAwDgYDVQQIDAdBcml6b25hMRAwDgYDVQQHDAdQ +aG9lbml4MSAwHgYDVQQKDBdBbm9ueW1vdXMgQml0IEZhY3RvcmllczEUMBIGA1UE +CwwLRW5naW5lZXJpbmcxGjAYBgNVBAMMEXBnLXRlc3QubG9jYWxob3N0MSUwIwYJ +KoZIhvcNAQkBFhZmYWtlQHBnLXRlc3QubG9jYWxob3N0MCAXDTIzMDIwNDIwMDEx +MFoYDzIwNzcxMjAxMjAwMTEwWjCBrDELMAkGA1UEBhMCVVMxEDAOBgNVBAgMB0Fy +aXpvbmExEDAOBgNVBAcMB1Bob2VuaXgxIDAeBgNVBAoMF0Fub255bW91cyBCaXQg +RmFjdG9yaWVzMRQwEgYDVQQLDAtFbmdpbmVlcmluZzEaMBgGA1UEAwwRcGctdGVz +dC5sb2NhbGhvc3QxJTAjBgkqhkiG9w0BCQEWFmZha2VAcGctdGVzdC5sb2NhbGhv +c3QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCnoLq3d7HCDHlBCN6h +ba+1rBKX1IyGw0p/AZWH3uBK0EIc2pkO47qefmqncabOQMgj4Hy4Ak97nOcdKWqP +aDXcSIUQ15oXcb2XW7NpYYS+4XJTayrHeW7+knPohISGWiCSsbP2KeYxwto0vObh +dtdH/2mN/dAqrvE8fQJpyNLN4D2B8CYKMokz8PGStZ7Iq8Em5UJlsM9sIe1LUwm8 +BxTQMv3B18+vj7Y/itWPMGBvNTM9RcsNP97ACT3hok9kBYC+02t1teNK8gn8JYF5 +0u3IKkc9yFHBo50muRlgi1hGpg/tIoyvB11NutD/vSwIcJL1irtfo070401WuAuF +VCbdRrcWwbIpxT9vH7ERhCwCwImPs+xZ1GEiZT/dt0eo+hvDIpFuyxrmLPWuvB1D +R4VRqS/6F7eepctSqao6zhGuszk69Gx78kKLk3nQ2VFeSKLHa73T28J09AXO+KJV +BuSmrNKZ6M3Q8XcZgWlzT5zscOnvwn1gR35Yy/HXosAFXL0qrl5hBsZA6ZpbMfTV +8f6He/EacDgb0CtcY8v3c/T7O02Ecl9cGDlvA3PLlxxriujLpA3Hzsz/6fbeIQcT +gGf266VgmxfHa6tQ4A31YIlMphQtN+4gKFYhEPLlx5hVx50eK6FE73RbIaybAgsC +gBXSN5sOiwwrpD6Qig4zYNPWfwIDAQABo1AwTjAdBgNVHQ4EFgQUy0r/yzk92MJZ +skVC5HrZ76nJKHkwHwYDVR0jBBgwFoAUy0r/yzk92MJZskVC5HrZ76nJKHkwDAYD +VR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAFYbhdj3Tz1E17iXe6dGQluCy +Fdo4PTMO7MwHhrpsscWdFpzJq9dtcWwLyGYIy111WNfB4AHVg8e13Ula5B9mi2CK +7kJjRZ+fFPuoBOG+qhXurf/yDhUwavF/forTCDiL58wc6QzGxp4TmkVyZzus2ryj +WmrgkLYMSzLNbWor/kLZzGh5OCUtLFXjL4EJn4NskbeOPvTotcmsOlokNryiH/t6 +ploi0TCL8JjdVblT1uPFtytEiheySJt3SZvL7tQhDBZfhNeup45f1bpQCtPGqqPd +9aTwSaatXNWfIltBpWMiyaj+udD7hntee0pD6iPdXh13knKOwhHzLET4OHEAPZGj +V4hZly5acthz6Xu9WLCznEo9/CZ1pyltKFP2Cx3xpkoGt8GQ3QiLdNvpox0xCVYM +8kQ9XGW3lEdZ+zl02flaN/Mah24RzDFAlceapSJLGg47Lrct+QWNuOo0LlAkA6Ir +XD96B4pjcfHmM1Qg0FROWed0UuDnnqFxM+4tyEnnPfhd6lgkQA8oVNJg8sgSm+Tl +NKdWyaylxx8ElI3e1ebzmfuY+J/DvlCbVd+7ZcPLAtsMqWIFWkWf2fXiLBWAll0Q +wqnIFRifRR6wFjSW2Re3gv64ShYWxqhRYztUSKzDFqJCmOyca/Ou4Yvfo2RJtiMk +kD4TZkFt1F7QewUFoMI= -----END CERTIFICATE----- """ From b606a8fae57f8b2f8b9ea20165f59c6d1567b08c Mon Sep 17 00:00:00 2001 From: James William Pye Date: Wed, 8 Feb 2023 18:18:20 -0700 Subject: [PATCH 091/109] Force password_encryption to md5 while SCRAM is unsupported. --- postgresql/test/test_connect.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/postgresql/test/test_connect.py b/postgresql/test/test_connect.py index ed587956..35ae98f1 100644 --- a/postgresql/test/test_connect.py +++ b/postgresql/test/test_connect.py @@ -80,6 +80,11 @@ def configure_cluster(self): if has_ipv6: listen_addresses += ',::1' + if self.cluster.installation.version_info >= (10, 0): + pwe = 'md5' + else: + pwe = 'on' + self.cluster.settings.update(dict( port = str(self.cluster_port), max_connections = '6', @@ -87,6 +92,7 @@ def configure_cluster(self): listen_addresses = listen_addresses, log_destination = 'stderr', log_min_messages = 'FATAL', + password_encryption = pwe, )) if self.disable_replication: From d36f3f824f46c94c29b068ce5f8084d35da3d970 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Thu, 9 Feb 2023 14:41:13 -0700 Subject: [PATCH 092/109] Add howto label link to Documentation section. --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a987adf4..912aaebf 100644 --- a/README.md +++ b/README.md @@ -50,9 +50,10 @@ with db.xact(): ### Documentation -http://py-postgresql.readthedocs.io +- https://py-postgresql.readthedocs.io +- https://github.com/python-postgres/fe/issues?q=label%3Ahowto ### Related -- http://postgresql.org -- http://python.org +- https://postgresql.org +- https://python.org From cc3dac147129129f76749f3c22791d4b4b806e0d Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 10 Feb 2023 07:05:20 -0700 Subject: [PATCH 093/109] Isolate security and startup initialization to allow normal ordering(super then sub). --- postgresql/driver/pq3.py | 52 ++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/postgresql/driver/pq3.py b/postgresql/driver/pq3.py index f0b7ef3e..6a778bd0 100644 --- a/postgresql/driver/pq3.py +++ b/postgresql/driver/pq3.py @@ -2781,24 +2781,27 @@ def socket_factory_sequence(self): def __init__(self, connect_timeout : int = None, server_encoding = None, - sslmode : ('allow', 'prefer', 'require', 'disable') = None, - sslcrtfile = None, - sslkeyfile = None, - sslrootcrtfile = None, - sslrootcrlfile = None, driver = None, **kw ): super().__init__(**kw) + self._security(kw) self.driver = driver - self.server_encoding = server_encoding self.connect_timeout = connect_timeout - self.sslmode = sslmode - self.sslkeyfile = sslkeyfile - self.sslcrtfile = sslcrtfile - self.sslrootcrtfile = sslrootcrtfile - self.sslrootcrlfile = sslrootcrlfile + + def _security(self, parameters): + self.sslmode = parameters.get('sslmode') or None + self.sslkeyfile = parameters.get('sslkeyfile') or None + self.sslcrtfile = parameters.get('sslcrtfile') or None + self.sslrootcrtfile = parameters.get('sslrootcrtfile') or None + self.sslrootcrlfile = parameters.get('sslrootcrlfile') or None + + self._socket_secure = { + 'keyfile': self.sslkeyfile, + 'certfile': self.sslcrtfile, + 'ca_certs': self.sslrootcrtfile, + } if self.sslrootcrlfile is not None: pg_exc.IgnoredClientParameterWarning( @@ -2806,6 +2809,7 @@ def __init__(self, creator = self, ).emit() + def _startup(self): # Startup message parameters. tnkw = { 'client_min_messages' : 'WARNING', @@ -2822,21 +2826,17 @@ def __init__(self, ) tnkw.update(s) + # Postgres defaults the database identifier to the user. tnkw['user'] = self.user if self.database is not None: tnkw['database'] = self.database + # Encode startup arguments. + # The server_encoding hint is strictly for str() values. se = self.server_encoding or 'utf-8' - ## - # Attempt to accommodate for literal treatment of startup data. - ## self._startup_parameters = tuple([ - # All keys go in utf-8. However, ascii would probably be good enough. ( k.encode('utf-8'), - # If it's a str(), encode in the hinted server_encoding. - # Otherwise, convert the object(int, float, bool, etc) into a string - # and treat it as utf-8. v.encode(se) if type(v) is str else str(v).encode('utf-8') ) for k, v in tnkw.items() @@ -2865,15 +2865,17 @@ def socket_factory_params(self, host, port, ipv, **kw): raise TypeError("'port' is a required keyword and cannot be 'None'") return {'socket_create': (self.address_family, socket.SOCK_STREAM), - 'socket_connect': (host, int(port))} + 'socket_connect': (host, int(port)), + 'socket_secure': self._socket_secure} def __init__(self, host, port, ipv, **kw): + super().__init__(**kw) params = self.socket_factory_params(host, port, ipv, **kw) self.host, self.port = params['socket_connect'] # constant socket connector self._socketcreator = self.create_socket_factory(**params) self._socketcreators = (self._socketcreator,) - super().__init__(**kw) + self._startup() class IP4(IPConnector): """ @@ -2917,15 +2919,17 @@ def socket_factory_params(self, unix): raise TypeError("'unix' is a required keyword and cannot be 'None'") return {'socket_create': (socket.AF_UNIX, socket.SOCK_STREAM), - 'socket_connect': unix} + 'socket_connect': unix, + 'socket_secure': self._socket_secure} def __init__(self, unix = None, **kw): + super().__init__(**kw) params = self.socket_factory_params(unix) self.unix = params['socket_connect'] # constant socket connector self._socketcreator = self.create_socket_factory(**params) self._socketcreators = (self._socketcreator,) - super().__init__(**kw) + self._startup() class Host(SocketConnector): """ @@ -2959,6 +2963,8 @@ def __init__(self, address_family = None, **kw ): + super().__init__(**kw) + if host is None: raise TypeError("'host' is a required keyword") if port is None: @@ -2977,7 +2983,7 @@ def __init__(self, raise TypeError("unknown IP version selected: 'ipv' = " + repr(ipv)) self.host = host self.port = port - super().__init__(**kw) + self._startup() class Driver(pg_api.Driver): def _e_metas(self): From 52152de4d18d81a4634a117c5ea9f5e05b1893d6 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 10 Feb 2023 07:08:57 -0700 Subject: [PATCH 094/109] Correct connect tests for modern Python ssl implementations(self-signed certificate exceptions). --- postgresql/test/test_connect.py | 61 +++++++++++++++++++---------- postgresql/test/test_ssl_connect.py | 5 ++- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/postgresql/test/test_connect.py b/postgresql/test/test_connect.py index 35ae98f1..22834a84 100644 --- a/postgresql/test/test_connect.py +++ b/postgresql/test/test_connect.py @@ -49,6 +49,9 @@ class TestCaseWithCluster(unittest.TestCase): postgresql.driver *interface* tests. """ installation = default_installation + @property + def _crt(self): + return self.params.get('sslrootcrtfile') or None def __init__(self, *args, **kw): super().__init__(*args, **kw) @@ -119,6 +122,7 @@ def initialize_database(self): c = self.cluster.connection( user = 'test', database = 'template1', + sslrootcrtfile = self._crt, ) with c: if c.prepare( @@ -128,13 +132,15 @@ def initialize_database(self): c.execute('create database test') def connection(self, *args, **kw): - return self.cluster.connection(*args, user = 'test', **kw) + return self.cluster.connection(*args, user = 'test', **self.params, **kw) def drop_cluster(self): if self.cluster.initialized(): self.cluster.drop() def run(self, *args, **kw): + self.params = {} + if 'PGINSTALLATION' not in os.environ: # Expect tests to show skipped. return super().run(*args, **kw) @@ -174,7 +180,6 @@ class test_connect(TestCaseWithCluster): ip6 = '::1' ip4 = '127.0.0.1' host = 'localhost' - params = {} cluster_path_suffix = '_test_connect' mk_common_users = """ @@ -228,19 +233,21 @@ def configure_cluster(self): def initialize_database(self): super().initialize_database() - with self.cluster.connection(user = 'test') as db: + with self.connection() as db: db.execute(self.mk_common_users) if self.check_crypt_user: db.execute(self.mk_crypt_user) @unittest.skipIf(default_installation is None, "no installation provided by environment") def test_pg_open_SQL_ASCII(self): - # postgresql.open host, port = self.cluster.address() + dbctx = self.params + # test simple locators.. with pg_open( 'pq://' + 'md5:' + 'md5_password@' + host + ':' + str(port) \ - + '/test?client_encoding=SQL_ASCII' + + '/test?client_encoding=SQL_ASCII', + **dbctx ) as db: self.assertEqual(db.prepare('select 1')(), [(1,)]) self.assertEqual(db.settings['client_encoding'], 'SQL_ASCII') @@ -249,66 +256,78 @@ def test_pg_open_SQL_ASCII(self): @unittest.skipIf(default_installation is None, "no installation provided by environment") def test_pg_open_keywords(self): host, port = self.cluster.address() - # straight test, no IRI + dbctx = self.params + + # Keywords only, no indicator. with pg_open( user = 'md5', password = 'md5_password', host = host, port = port, - database = 'test' + database = 'test', + **dbctx, ) as db: self.assertEqual(db.prepare('select 1')(), [(1,)]) - self.assertTrue(db.closed) - # composite test + + # Keyword and indicator source. with pg_open( "pq://md5:md5_password@", host = host, port = port, - database = 'test' + database = 'test', + **dbctx, ) as db: self.assertEqual(db.prepare('select 1')(), [(1,)]) - # override test + + # Keyword override. with pg_open( "pq://md5:foobar@", password = 'md5_password', host = host, port = port, - database = 'test' + database = 'test', + **dbctx, ) as db: self.assertEqual(db.prepare('select 1')(), [(1,)]) - # and, one with some settings + + # Settings override. with pg_open( "pq://md5:foobar@?search_path=ieeee", password = 'md5_password', host = host, port = port, database = 'test', - settings = {'search_path' : 'public'} + settings = {'search_path' : 'public'}, + **dbctx, ) as db: self.assertEqual(db.prepare('select 1')(), [(1,)]) self.assertEqual(db.settings['search_path'], 'public') @unittest.skipIf(default_installation is None, "no installation provided by environment") def test_pg_open(self): - # postgresql.open host, port = self.cluster.address() + dbctx = self.params + # test simple locators.. with pg_open( 'pq://' + 'md5:' + 'md5_password@' + host + ':' + str(port) \ - + '/test' + + '/test', + **dbctx, ) as db: self.assertEqual(db.prepare('select 1')(), [(1,)]) self.assertTrue(db.closed) with pg_open( 'pq://' + 'password:' + 'password_password@' + host + ':' + str(port) \ - + '/test' + + '/test', + **dbctx, ) as db: self.assertEqual(db.prepare('select 1')(), [(1,)]) self.assertTrue(db.closed) with pg_open( - 'pq://' + 'trusted@' + host + ':' + str(port) + '/test' + 'pq://' + 'trusted@' + host + ':' + str(port) + '/test', + **dbctx, ) as db: self.assertEqual(db.prepare('select 1')(), [(1,)]) self.assertTrue(db.closed) @@ -324,7 +343,7 @@ def test_pg_open(self): os.environ['PGPORT'] = str(port) os.environ['PGDATABASE'] = 'test' # No arguments, the environment provided everything. - with pg_open() as db: + with pg_open(**dbctx) as db: self.assertEqual(db.prepare('select 1')(), [(1,)]) self.assertEqual(db.prepare('select current_user').first(), 'md5') self.assertTrue(db.closed) @@ -354,7 +373,7 @@ def test_pg_open(self): try: os.environ['PGSERVICE'] = 'myserv' os.environ['PGSYSCONFDIR'] = os.getcwd() - with pg_open() as db: + with pg_open(**dbctx) as db: self.assertEqual(db.prepare('select 1')(), [(1,)]) self.assertEqual(db.prepare('select current_user').first(), 'password') self.assertEqual(db.settings['search_path'], 'public') @@ -505,6 +524,7 @@ def test_password_connect(self): user = 'password', password = 'password_password', database = 'test', + sslrootcrtfile = self._crt, ) with c: self.assertEqual(c.prepare('select current_user').first(), 'password') @@ -524,6 +544,7 @@ def test_trusted_connect(self): def test_Unix_connect(self): if not has_unix_sock: return + unix_domain_socket = os.path.join( self.cluster.data_directory, '.s.PGSQL.' + self.cluster.settings['port'] diff --git a/postgresql/test/test_ssl_connect.py b/postgresql/test/test_ssl_connect.py index f9b90a6d..30f511d0 100644 --- a/postgresql/test/test_ssl_connect.py +++ b/postgresql/test/test_ssl_connect.py @@ -141,12 +141,15 @@ def configure_cluster(self): os.chmod(key_file, 0o700) os.chmod(crt_file, 0o700) + self.params['sslrootcrtfile'] = crt_file + def initialize_database(self): if not has_ssl: return super().initialize_database() - with self.cluster.connection(user = 'test') as db: + # Setup TLS users. + with self.cluster.connection(user = 'test', **self.params) as db: db.execute( """ CREATE USER nossl; From fb1ed35a2436f8cd1d2db69b754afb2aa323e543 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 10 Feb 2023 07:10:58 -0700 Subject: [PATCH 095/109] Heed ssl.wrap_socket deprecation warning and defer ssl import. --- postgresql/python/socket.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/postgresql/python/socket.py b/postgresql/python/socket.py index 6cdffdca..6587d4c1 100644 --- a/postgresql/python/socket.py +++ b/postgresql/python/socket.py @@ -6,7 +6,6 @@ import random import socket import errno -import ssl __all__ = ['find_available_port', 'SocketFactory'] @@ -49,14 +48,29 @@ def fatal_exception_message(typ, err) -> (str, None): return None return getattr(err, 'strerror', '') - def secure(self, socket : socket.socket) -> ssl.SSLSocket: + @property + def _security_context(self): + if self._security_context_ii is None: + from ssl import SSLContext, PROTOCOL_TLS_CLIENT + ctx = self._security_context_ii = SSLContext(PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + + cf = self.socket_secure.get('certfile') + kf = self.socket_secure.get('keyfile') + if cf is not None: + self._security_context_ii.load_cert_chain(cf, keyfile=kf) + + ca = self.socket_secure.get('ca_certs') + if ca is not None: + self._security_context_ii.load_verify_locations(ca) + + return self._security_context_ii + + def secure(self, socket: socket.socket): """ Secure a socket with SSL. """ - if self.socket_secure is not None: - return ssl.wrap_socket(socket, **self.socket_secure) - else: - return ssl.wrap_socket(socket) + return self._security_context.wrap_socket(socket) def __call__(self, timeout = None): s = socket.socket(*self.socket_create) @@ -73,10 +87,12 @@ def __init__(self, socket_create, socket_connect, socket_secure = None, + socket_security_context = None ): + self._security_context_ii = socket_security_context self.socket_create = socket_create self.socket_connect = socket_connect - self.socket_secure = socket_secure + self.socket_secure = socket_secure or {} def __str__(self): return 'socket' + repr(self.socket_connect) From e0faeb35aecdd9813bee3640bd538315528c840b Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 10 Feb 2023 08:11:54 -0700 Subject: [PATCH 096/109] Include installation from a sparse checkout excluding the usual packaging files. --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index 912aaebf..18c712dc 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,19 @@ From a clone: $ cd fe $ python3 ./setup.py install # Or use in-place without installation(PYTHONPATH). +Direct from a sparse checkout: + + export BRANCH=v1.3 + export TARGET="$(pwd)/py-packages" + export PYTHONPATH="$PYTHONPATH:$TARGET" + git clone --origin=pypg-frontend --branch=$BRANCH \ + --sparse --filter=blob:none --no-checkout --depth=1 \ + https://github.com/python-postgres/fe.git "$TARGET" + pushd "$TARGET" + git sparse-checkout set --no-cone postgresql + git switch $BRANCH + popd; unset TARGET BRANCH + ### Basic Usage ```python @@ -48,6 +61,10 @@ with db.xact(): print(x) ``` +REPL with connection bound to `db` builtin: + + python3 -m postgresql.bin.pg_python -I 'pq://postgres@localhost:5423/postgres' + ### Documentation - https://py-postgresql.readthedocs.io From 3bee7a5e34d63edbcb1903706aaaaf413d74fe59 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 10 Feb 2023 08:21:10 -0700 Subject: [PATCH 097/109] Add PostgREST link. --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 18c712dc..d5fcb38a 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,11 @@ py-postgresql is a Python 3 package providing modules for working with PostgreSQL. Primarily, a high-level driver for querying databases. -For a high performance async interface, MagicStack's asyncpg -http://github.com/MagicStack/asyncpg should be considered. +While py-postgresql is still usable for many purposes, asyncpg and PostgREST are +likely more suitable for most applications: + + - http://github.com/MagicStack/asyncpg + - https://postgrest.org/ py-postgresql, currently, does not have direct support for high-level async interfaces provided by recent versions of Python. Future versions may change this. From 265a113d5f20cdc8a77ff7e2923a81a037b1adf8 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 10 Feb 2023 08:22:44 -0700 Subject: [PATCH 098/109] Correct syntax. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d5fcb38a..b962a0f4 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ Primarily, a high-level driver for querying databases. While py-postgresql is still usable for many purposes, asyncpg and PostgREST are likely more suitable for most applications: - - http://github.com/MagicStack/asyncpg - - https://postgrest.org/ +- http://github.com/MagicStack/asyncpg +- https://postgrest.org py-postgresql, currently, does not have direct support for high-level async interfaces provided by recent versions of Python. Future versions may change this. From a19a761ba0c79c9c95cd8b53c079161af61e3757 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 10 Feb 2023 08:38:26 -0700 Subject: [PATCH 099/109] Relocate the v2.0 warning. --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b962a0f4..94ce21af 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Primarily, a high-level driver for querying databases. While py-postgresql is still usable for many purposes, asyncpg and PostgREST are likely more suitable for most applications: -- http://github.com/MagicStack/asyncpg +- https://github.com/MagicStack/asyncpg - https://postgrest.org py-postgresql, currently, does not have direct support for high-level async @@ -14,6 +14,9 @@ interfaces provided by recent versions of Python. Future versions may change thi ### Advisory +In v2.0, support for older versions of PostgreSQL and Python will be removed. +If you have automated installations using PyPI, make sure that they specify a major version. + In v1.3, `postgresql.driver.dbapi20.connect` will now raise `ClientCannotConnectError` directly. Exception traps around connect should still function, but the `__context__` attribute on the error instance will be `None` in the usual failure case as it is no longer @@ -21,9 +24,6 @@ incorrectly chained. Trapping `ClientCannotConnectError` ahead of `Error` should allow both cases to co-exist in the event that data is being extracted from the `ClientCannotConnectError`. -In v2.0, support for older versions of PostgreSQL and Python will be removed. -If you have automated installations using PyPI, make sure that they specify a major version. - ### Installation Using PyPI.org: From 2eae8509159ed3c6d6fdae34aabb95170356dd66 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Fri, 10 Feb 2023 11:14:11 -0700 Subject: [PATCH 100/109] Link Project Future issue. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 94ce21af..48a99f91 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ likely more suitable for most applications: py-postgresql, currently, does not have direct support for high-level async interfaces provided by recent versions of Python. Future versions may change this. +- [Project Future](https://github.com/python-postgres/fe/issues/124) + ### Advisory In v2.0, support for older versions of PostgreSQL and Python will be removed. From b78bb3a255d1e52fa3a3a263dd6d38aabce06db2 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sat, 11 Feb 2023 06:01:51 -0700 Subject: [PATCH 101/109] Identify shell as bash and adjust sparse checkout to use a subshell. --- README.md | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 48a99f91..6cfe978d 100644 --- a/README.md +++ b/README.md @@ -28,28 +28,36 @@ the `ClientCannotConnectError`. ### Installation -Using PyPI.org: +Using `pip` and [PyPI](https://PyPI.org): - $ pip install py-postgresql +```bash +python3 -m pip install py-postgresql +``` -From a clone: +From [GitHub](https://github.com) using a full clone: - $ git clone https://github.com/python-postgres/fe.git - $ cd fe - $ python3 ./setup.py install # Or use in-place without installation(PYTHONPATH). +```bash +git clone https://github.com/python-postgres/fe.git +cd fe +python3 ./setup.py install +``` -Direct from a sparse checkout: +From [GitHub](https://github.com) using a sparse checkout: - export BRANCH=v1.3 - export TARGET="$(pwd)/py-packages" - export PYTHONPATH="$PYTHONPATH:$TARGET" +```bash +TARGET="$(pwd)/py-packages" +export PYTHONPATH="$PYTHONPATH:$TARGET" +(set -e + BRANCH=v1.3 git clone --origin=pypg-frontend --branch=$BRANCH \ --sparse --filter=blob:none --no-checkout --depth=1 \ https://github.com/python-postgres/fe.git "$TARGET" - pushd "$TARGET" + cd "$TARGET" git sparse-checkout set --no-cone postgresql git switch $BRANCH - popd; unset TARGET BRANCH +) +unset TARGET +``` ### Basic Usage @@ -68,7 +76,9 @@ with db.xact(): REPL with connection bound to `db` builtin: - python3 -m postgresql.bin.pg_python -I 'pq://postgres@localhost:5423/postgres' +```bash +python3 -m postgresql.bin.pg_python -I 'pq://postgres@localhost:5423/postgres' +``` ### Documentation From c92675be1a14ae3058d16d51a1cc796c935cc6a2 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sat, 11 Feb 2023 06:30:04 -0700 Subject: [PATCH 102/109] Not a hobgoblin. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6cfe978d..5cdb2f75 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ the `ClientCannotConnectError`. ### Installation -Using `pip` and [PyPI](https://PyPI.org): +From [PyPI](https://PyPI.org) using `pip`: ```bash python3 -m pip install py-postgresql From 546b6dce8c726df7ff3477311df2f13323dd5d69 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sun, 12 Feb 2023 14:09:22 -0700 Subject: [PATCH 103/109] Use git-select, frighteningly, from curl; rewrite sparse checkout installation to be limited to the subshell. --- README.md | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 5cdb2f75..0425d8a1 100644 --- a/README.md +++ b/README.md @@ -42,21 +42,18 @@ cd fe python3 ./setup.py install ``` -From [GitHub](https://github.com) using a sparse checkout: +From [GitHub](https://github.com) using a temporary installation scoped to a subshell: ```bash -TARGET="$(pwd)/py-packages" -export PYTHONPATH="$PYTHONPATH:$TARGET" -(set -e - BRANCH=v1.3 - git clone --origin=pypg-frontend --branch=$BRANCH \ - --sparse --filter=blob:none --no-checkout --depth=1 \ - https://github.com/python-postgres/fe.git "$TARGET" - cd "$TARGET" - git sparse-checkout set --no-cone postgresql - git switch $BRANCH -) -unset TARGET +(PYTMPPKG="$(mktemp -d)" +export PYTHONPATH="$PYTHONPATH:$PYTMPPKG" +curl -s https://raw.githubusercontent.com/jwp/git-select/main/git-select.py | \ + python3 /dev/stdin \ + https://github.com/python-postgres/fe master "postgresql/./$PYTMPPKG/" +python3 -c "import postgresql.project as pj; print(); print('py-postgresql:', pj.version)" +export pg_console="postgresql.bin.pg_python" +echo ': python3 -m $pg_console' +$SHELL) ``` ### Basic Usage From e4f85bc137ae78444ac4f540ed7c595b9c525602 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Tue, 14 Mar 2023 06:08:42 -0700 Subject: [PATCH 104/109] Note upcoming big changes. --- README.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 0425d8a1..6e947a0a 100644 --- a/README.md +++ b/README.md @@ -16,15 +16,14 @@ interfaces provided by recent versions of Python. Future versions may change thi ### Advisory -In v2.0, support for older versions of PostgreSQL and Python will be removed. +In v2.0, many, potentially breaking, changes are planned. If you have automated installations using PyPI, make sure that they specify a major version. -In v1.3, `postgresql.driver.dbapi20.connect` will now raise `ClientCannotConnectError` directly. -Exception traps around connect should still function, but the `__context__` attribute -on the error instance will be `None` in the usual failure case as it is no longer -incorrectly chained. Trapping `ClientCannotConnectError` ahead of `Error` should -allow both cases to co-exist in the event that data is being extracted from -the `ClientCannotConnectError`. +- Support for older versions of PostgreSQL and Python will be removed. This will allow the driver +to defer version parsing fixing #109, and better prepare for future versions. +- The connection establishment strategy will be simplified to only performing one attempt. `sslmode` +parameter should be considered deprecated. See #122 and #75. +- StoredProcedure will be removed. See #80. ### Installation From 9490eae47c1f2ddcd987e6fff7c4fd9de5e6fc97 Mon Sep 17 00:00:00 2001 From: James William Pye Date: Tue, 14 Mar 2023 06:19:03 -0700 Subject: [PATCH 105/109] Update README.md Correct issue links and expand strategy bullet. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6e947a0a..eb9c1fbc 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,10 @@ In v2.0, many, potentially breaking, changes are planned. If you have automated installations using PyPI, make sure that they specify a major version. - Support for older versions of PostgreSQL and Python will be removed. This will allow the driver -to defer version parsing fixing #109, and better prepare for future versions. +to defer version parsing fixing (https://github.com/python-postgres/fe/issues/109), and better prepare for future versions. - The connection establishment strategy will be simplified to only performing one attempt. `sslmode` -parameter should be considered deprecated. See #122 and #75. -- StoredProcedure will be removed. See #80. +parameter should be considered deprecated. v1.4 will provide a new security parameter implying `sslmode=require`. See (https://github.com/python-postgres/fe/issues/122) and (https://github.com/python-postgres/fe/issues/75). +- StoredProcedure will be removed. See (https://github.com/python-postgres/fe/issues/80). ### Installation From cf191e7558681fa0e5e639f3026bd6885b01adea Mon Sep 17 00:00:00 2001 From: James William Pye Date: Tue, 14 Mar 2023 06:28:30 -0700 Subject: [PATCH 106/109] Remove the not so useful temporary installation; too long. --- README.md | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/README.md b/README.md index eb9c1fbc..db9fffc7 100644 --- a/README.md +++ b/README.md @@ -41,20 +41,6 @@ cd fe python3 ./setup.py install ``` -From [GitHub](https://github.com) using a temporary installation scoped to a subshell: - -```bash -(PYTMPPKG="$(mktemp -d)" -export PYTHONPATH="$PYTHONPATH:$PYTMPPKG" -curl -s https://raw.githubusercontent.com/jwp/git-select/main/git-select.py | \ - python3 /dev/stdin \ - https://github.com/python-postgres/fe master "postgresql/./$PYTMPPKG/" -python3 -c "import postgresql.project as pj; print(); print('py-postgresql:', pj.version)" -export pg_console="postgresql.bin.pg_python" -echo ': python3 -m $pg_console' -$SHELL) -``` - ### Basic Usage ```python From 260d4d42e0c3a2951fe9543dc4e18d93c12950ec Mon Sep 17 00:00:00 2001 From: Rainer Hurling Date: Mon, 1 May 2023 14:15:48 +0200 Subject: [PATCH 107/109] buffer.c: Fix incompatible pointer to integer conversion Part 1 of 2 changes --- postgresql/port/_optimized/buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/port/_optimized/buffer.c b/postgresql/port/_optimized/buffer.c index 7f46d2d1..cb81b2f9 100644 --- a/postgresql/port/_optimized/buffer.c +++ b/postgresql/port/_optimized/buffer.c @@ -587,7 +587,7 @@ PyTypeObject pq_message_stream_Type = { sizeof(struct p_buffer), /* tp_basicsize */ 0, /* tp_itemsize */ p_dealloc, /* tp_dealloc */ - NULL, /* tp_print */ + 0, /* tp_print */ NULL, /* tp_getattr */ NULL, /* tp_setattr */ NULL, /* tp_compare */ From e24e486cd4bf08a36c0c358300ad80817aa279a0 Mon Sep 17 00:00:00 2001 From: Rainer Hurling Date: Mon, 1 May 2023 14:16:59 +0200 Subject: [PATCH 108/109] wirestate.c: Fix incompatible pointer to integer conversion Part 2 of 2 changes --- postgresql/port/_optimized/wirestate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/port/_optimized/wirestate.c b/postgresql/port/_optimized/wirestate.c index 7947e5cb..74a9aca4 100644 --- a/postgresql/port/_optimized/wirestate.c +++ b/postgresql/port/_optimized/wirestate.c @@ -248,7 +248,7 @@ PyTypeObject WireState_Type = { sizeof(struct wirestate), /* tp_basicsize */ 0, /* tp_itemsize */ ws_dealloc, /* tp_dealloc */ - NULL, /* tp_print */ + 0, /* tp_print */ NULL, /* tp_getattr */ NULL, /* tp_setattr */ NULL, /* tp_compare */ From 6ce98853b28b0389e4325aacbce81dd4b62132ed Mon Sep 17 00:00:00 2001 From: James William Pye Date: Sun, 9 Mar 2025 07:18:39 -0700 Subject: [PATCH 109/109] Correct escapes for 3.13; fixes #128. --- postgresql/protocol/element3.py | 2 +- postgresql/python/command.py | 2 +- postgresql/types/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/postgresql/protocol/element3.py b/postgresql/protocol/element3.py index e5a95781..1f8579e2 100644 --- a/postgresql/protocol/element3.py +++ b/postgresql/protocol/element3.py @@ -263,7 +263,7 @@ def extract_command(self): Strip all the *surrounding* digits and spaces from the command tag, and return that string. """ - return self.data.strip(b'\c\n\t 0123456789') or None + return self.data.strip(b'\r\n\t 0123456789') or None class Null(EmptyMessage): """ diff --git a/postgresql/python/command.py b/postgresql/python/command.py index 18685c22..d88b1541 100644 --- a/postgresql/python/command.py +++ b/postgresql/python/command.py @@ -220,7 +220,7 @@ def __init__(self, *args, **kw): self.register_backslash(r'\?', self.showhelp, "Show this help message.") self.register_backslash(r'\set', self.bs_set, - "Configure environment variables. \set without arguments to show all") + "Configure environment variables. \\set without arguments to show all") self.register_backslash(r'\E', self.bs_E, "Edit a file or a temporary script.") self.register_backslash(r'\i', self.bs_i, diff --git a/postgresql/types/__init__.py b/postgresql/types/__init__.py index 690e19f3..12b2543e 100644 --- a/postgresql/types/__init__.py +++ b/postgresql/types/__init__.py @@ -610,7 +610,7 @@ def column_names(self, get0 = get0, get1 = get1): def transform(self, *args, **kw): """ Make a new Row after processing the values with the callables associated - with the values either by index, \*args, or my column name, \*\*kw. + with the values either by index, *args, or my column name, **kw. >>> r=Row.from_sequence({'col1':0,'col2':1}, (1,'two')) >>> r.transform(str)