From 7d6fdb95b8093c969e584fe32044030e431595a4 Mon Sep 17 00:00:00 2001 From: rdrg109 <> Date: Tue, 17 Aug 2021 21:46:28 -0500 Subject: [PATCH 1/4] Execute "moveTableToSchema" if table was succesfully created --- load_into_pg.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/load_into_pg.py b/load_into_pg.py index 32c05aa..2185538 100755 --- a/load_into_pg.py +++ b/load_into_pg.py @@ -402,10 +402,18 @@ def moveTableToSchema(table, schemaName, dbConnectionParam): handleTable( table, args.insert_json, args.foreign_keys, args.file, dbConnectionParam ) + + # Once "handleTable" has created the table in the "public" + # schema, move the table to the given schema if and only if + # the user specified an schema as an argument for the script + # and the schema name is different than "public". + + if args.schema_name != "public": + moveTableToSchema(table, args.schema_name, dbConnectionParam) + else: six.print_("Cancelled.") - if args.schema_name != "public": - moveTableToSchema(table, args.schema_name, dbConnectionParam) + exit(0) # load a project From d89b9ffe2192fdc90b92577f315b777a59d0eb20 Mon Sep 17 00:00:00 2001 From: rdrg109 <> Date: Fri, 20 Aug 2021 19:31:26 -0500 Subject: [PATCH 2/4] Function for getting the connection parameters added --- load_into_pg.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/load_into_pg.py b/load_into_pg.py index 2185538..b15faf1 100755 --- a/load_into_pg.py +++ b/load_into_pg.py @@ -46,6 +46,31 @@ def show_progress(block_num, block_size, total_size): file_part = None six.print_("") +def getConnectionParameters(): + """Get the parameters for the connection to the database.""" + + parameters = {} + + if args.dbname: + parameters['dbname'] = args.dbname + + if args.host: + parameters['host'] = args.host + + if args.port: + parameters['port'] = args.port + + if args.username: + parameters['user'] = args.username + + if args.password: + parameters['password'] = args.password + + if args.schema_name: + parameters['options'] = "-c search_path=" + args.schema_name + + return parameters + def buildConnectionString(dbname, mbHost, mbPort, mbUsername, mbPassword): dbConnectionParam = "dbname={}".format(dbname) From 2b1bbee1ec60268ddc6205c164b2fb25874c14be Mon Sep 17 00:00:00 2001 From: rdrg109 <> Date: Fri, 20 Aug 2021 19:31:59 -0500 Subject: [PATCH 3/4] Function buildConnectionString deleted This is because the parameters are now obtained through the function getConnectionParameters --- load_into_pg.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/load_into_pg.py b/load_into_pg.py index b15faf1..087f442 100755 --- a/load_into_pg.py +++ b/load_into_pg.py @@ -72,25 +72,6 @@ def getConnectionParameters(): return parameters -def buildConnectionString(dbname, mbHost, mbPort, mbUsername, mbPassword): - dbConnectionParam = "dbname={}".format(dbname) - - if mbPort is not None: - dbConnectionParam += " port={}".format(mbPort) - - if mbHost is not None: - dbConnectionParam += " host={}".format(mbHost) - - # TODO Is the escaping done here correct? - if mbUsername is not None: - dbConnectionParam += " user={}".format(mbUsername) - - # TODO Is the escaping done here correct? - if mbPassword is not None: - dbConnectionParam += " password={}".format(mbPassword) - return dbConnectionParam - - def _makeDefValues(keys): """Returns a dictionary containing None for all keys.""" return dict(((k, None) for k in keys)) From 5afa6714f26b9c0c75144262e81c32bc0725373d Mon Sep 17 00:00:00 2001 From: rdrg109 <> Date: Fri, 20 Aug 2021 19:35:25 -0500 Subject: [PATCH 4/4] dbConnectionParam and moveTableToSchema replaced by single function dbConnectionParam has been deleted because now all parameters are obtained by a single function which is called in the function handleTable. moveTableToSchema has been deleted because tables are now created in the specified schema. Therefore, there is no need to move the table after their creation. --- load_into_pg.py | 47 +++++------------------------------------------ 1 file changed, 5 insertions(+), 42 deletions(-) diff --git a/load_into_pg.py b/load_into_pg.py index 087f442..2d6cef5 100755 --- a/load_into_pg.py +++ b/load_into_pg.py @@ -180,7 +180,7 @@ def _getTableKeys(table): return keys -def handleTable(table, insertJson, createFk, mbDbFile, dbConnectionParam): +def handleTable(table, insertJson, createFk, mbDbFile): """Handle the table including the post/pre processing.""" keys = _getTableKeys(table) dbFile = mbDbFile if mbDbFile is not None else table + ".xml" @@ -199,7 +199,7 @@ def handleTable(table, insertJson, createFk, mbDbFile, dbConnectionParam): sys.exit(-1) try: - with pg.connect(dbConnectionParam) as conn: + with pg.connect(**getConnectionParameters()) as conn: with conn.cursor() as cur: try: with open(dbFile, "rb") as xml: @@ -279,29 +279,8 @@ def handleTable(table, insertJson, createFk, mbDbFile, dbConnectionParam): six.print_("Warning from the database.", file=sys.stderr) six.print_("pg.Warning: {0}".format(str(w)), file=sys.stderr) - -def moveTableToSchema(table, schemaName, dbConnectionParam): - try: - with pg.connect(dbConnectionParam) as conn: - with conn.cursor() as cur: - # create the schema - cur.execute("CREATE SCHEMA IF NOT EXISTS " + schemaName + ";") - conn.commit() - # move the table to the right schema - cur.execute("ALTER TABLE " + table + " SET SCHEMA " + schemaName + ";") - conn.commit() - except pg.Error as e: - six.print_("Error in dealing with the database.", file=sys.stderr) - six.print_("pg.Error ({0}): {1}".format(e.pgcode, e.pgerror), file=sys.stderr) - six.print_(str(e), file=sys.stderr) - except pg.Warning as w: - six.print_("Warning from the database.", file=sys.stderr) - six.print_("pg.Warning: {0}".format(str(w)), file=sys.stderr) - - ############################################################# - parser = argparse.ArgumentParser() parser.add_argument( "-t", @@ -390,10 +369,6 @@ def moveTableToSchema(table, schemaName, dbConnectionParam): except NameError: pass -dbConnectionParam = buildConnectionString( - args.dbname, args.host, args.port, args.username, args.password -) - # load given file in table if args.file and args.table: table = args.table @@ -404,19 +379,10 @@ def moveTableToSchema(table, schemaName, dbConnectionParam): specialRules[("Posts", "Body")] = "NULL" choice = input("This will drop the {} table. Are you sure [y/n]?".format(table)) + if len(choice) > 0 and choice[0].lower() == "y": handleTable( - table, args.insert_json, args.foreign_keys, args.file, dbConnectionParam - ) - - # Once "handleTable" has created the table in the "public" - # schema, move the table to the given schema if and only if - # the user specified an schema as an argument for the script - # and the schema name is different than "public". - - if args.schema_name != "public": - moveTableToSchema(table, args.schema_name, dbConnectionParam) - + table, args.insert_json, args.foreign_keys, args.file) else: six.print_("Cancelled.") @@ -467,7 +433,7 @@ def moveTableToSchema(table, schemaName, dbConnectionParam): for table in tables: six.print_("Load {0}.xml file".format(table)) - handleTable(table, args.insert_json, args.foreign_keys, None, dbConnectionParam) + handleTable(table, args.insert_json, args.foreign_keys, None) # remove file os.remove(table + ".xml") @@ -479,9 +445,6 @@ def moveTableToSchema(table, schemaName, dbConnectionParam): else: six.print_("Archive '{0}' deleted".format(filepath)) - if args.schema_name != "public": - for table in tables: - moveTableToSchema(table, args.schema_name, dbConnectionParam) exit(0) else: