From 42fef3406230b9d938cc05e11b24c03c1197dfab Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Sun, 21 Apr 2024 05:59:34 +0200 Subject: [PATCH 01/19] db: fix SQL TRANSACTION syntax compatible with all DB backends SQL TRANSACTION syntax: ``` BEGIN; ...; COMMIT; ``` --- python/grass/temporal/core.py | 6 +++--- scripts/db.dropcolumn/db.dropcolumn.py | 4 ++-- scripts/v.db.addcolumn/v.db.addcolumn.py | 4 ++-- scripts/v.db.dropcolumn/v.db.dropcolumn.py | 4 ++-- scripts/v.db.join/v.db.join.py | 6 ++---- scripts/v.db.univar/tests/conftest.py | 4 ++-- scripts/v.dissolve/tests/conftest.py | 4 ++-- scripts/v.dissolve/v.dissolve.py | 4 ++-- 8 files changed, 17 insertions(+), 19 deletions(-) diff --git a/python/grass/temporal/core.py b/python/grass/temporal/core.py index 68db2716ae5..ab7d2912d12 100644 --- a/python/grass/temporal/core.py +++ b/python/grass/temporal/core.py @@ -1560,7 +1560,7 @@ def fetchall(self): def execute_transaction(self, statement, mapset=None): """Execute a transactional SQL statement - The BEGIN and END TRANSACTION statements will be added automatically + The BEGIN and COMMIT statements will be added automatically to the sql statement :param statement: The executable SQL statement or SQL script @@ -1571,9 +1571,9 @@ def execute_transaction(self, statement, mapset=None): connected = True sql_script = "" - sql_script += "BEGIN TRANSACTION;\n" + sql_script += "BEGIN;\n" sql_script += statement - sql_script += "END TRANSACTION;" + sql_script += "COMMIT;" try: if self.dbmi.__name__ == "sqlite3": diff --git a/scripts/db.dropcolumn/db.dropcolumn.py b/scripts/db.dropcolumn/db.dropcolumn.py index 13d0f878beb..e6f67716279 100755 --- a/scripts/db.dropcolumn/db.dropcolumn.py +++ b/scripts/db.dropcolumn/db.dropcolumn.py @@ -120,14 +120,14 @@ def main(): coltypes = ", ".join(coltypes) cmds = [ - "BEGIN TRANSACTION", + "BEGIN;", "CREATE TEMPORARY TABLE ${table}_backup(${coldef})", "INSERT INTO ${table}_backup SELECT ${colnames} FROM ${table}", "DROP TABLE ${table}", "CREATE TABLE ${table}(${coldef})", "INSERT INTO ${table} SELECT ${colnames} FROM ${table}_backup", "DROP TABLE ${table}_backup", - "COMMIT", + "COMMIT;", ] tmpl = string.Template(";\n".join(cmds)) sql = tmpl.substitute(table=table, coldef=coltypes, colnames=colnames) diff --git a/scripts/v.db.addcolumn/v.db.addcolumn.py b/scripts/v.db.addcolumn/v.db.addcolumn.py index 8d8e322118c..ed4ca712bb7 100755 --- a/scripts/v.db.addcolumn/v.db.addcolumn.py +++ b/scripts/v.db.addcolumn/v.db.addcolumn.py @@ -99,7 +99,7 @@ def main(): driver = f["driver"] column_existing = grass.vector_columns(map, int(layer)).keys() - add_str = "BEGIN TRANSACTION\n" + add_str = "BEGIN;\n" pattern = re.compile(r"\s+") for col in columns: if not col: @@ -120,7 +120,7 @@ def main(): continue grass.verbose(_("Adding column <{}> to the table").format(col_name)) add_str += f'ALTER TABLE {table} ADD COLUMN "{col_name}" {col_type};\n' - add_str += "END TRANSACTION" + add_str += "COMMIT;" sql_file = grass.tempfile() rm_files.append(sql_file) cols_add_str = ",".join([col[0] for col in columns]) diff --git a/scripts/v.db.dropcolumn/v.db.dropcolumn.py b/scripts/v.db.dropcolumn/v.db.dropcolumn.py index b408c83395a..a05a14c4bc8 100755 --- a/scripts/v.db.dropcolumn/v.db.dropcolumn.py +++ b/scripts/v.db.dropcolumn/v.db.dropcolumn.py @@ -104,7 +104,7 @@ def main(): coltypes = ", ".join(coltypes) cmds = [ - "BEGIN TRANSACTION", + "BEGIN;", "CREATE TEMPORARY TABLE ${table}_backup (${coldef})", "INSERT INTO ${table}_backup SELECT ${colnames} FROM ${table}", "DROP TABLE ${table}", @@ -112,7 +112,7 @@ def main(): "INSERT INTO ${table} SELECT ${colnames} FROM ${table}_backup", "CREATE UNIQUE INDEX ${table}_cat ON ${table} (${keycol} )", "DROP TABLE ${table}_backup", - "COMMIT", + "COMMIT;", ] tmpl = string.Template(";\n".join(cmds)) sql = tmpl.substitute( diff --git a/scripts/v.db.join/v.db.join.py b/scripts/v.db.join/v.db.join.py index 104bee1b92f..22696ab753f 100755 --- a/scripts/v.db.join/v.db.join.py +++ b/scripts/v.db.join/v.db.join.py @@ -218,7 +218,6 @@ def main(): ) ) - update_str = "BEGIN TRANSACTION\n" for col in cols_to_update: cur_up_str = ( f"UPDATE {maptable} SET {col} = (SELECT {col} FROM " @@ -226,9 +225,8 @@ def main(): f"{otable}.{ocolumn}={maptable}.{column});\n" ) update_str += cur_up_str - update_str += "END TRANSACTION" - gs.debug(update_str, 1) - gs.verbose( + grass.debug(update_str, 1) + grass.verbose( _("Updating columns {columns} of vector map {map_name}...").format( columns=", ".join(cols_to_update.keys()), map_name=vector_map ) diff --git a/scripts/v.db.univar/tests/conftest.py b/scripts/v.db.univar/tests/conftest.py index 6edbc788018..336aace4f19 100644 --- a/scripts/v.db.univar/tests/conftest.py +++ b/scripts/v.db.univar/tests/conftest.py @@ -9,10 +9,10 @@ def updates_as_transaction(table, cat_column, column, cats, values): """Create SQL statement for categories and values for a given column""" - sql = ["BEGIN TRANSACTION"] + sql = ["BEGIN;"] for cat, value in zip(cats, values): sql.append(f"UPDATE {table} SET {column} = {value} WHERE {cat_column} = {cat};") - sql.append("END TRANSACTION") + sql.append("COMMIT;") return "\n".join(sql) diff --git a/scripts/v.dissolve/tests/conftest.py b/scripts/v.dissolve/tests/conftest.py index b74969999b1..b9956857e1d 100644 --- a/scripts/v.dissolve/tests/conftest.py +++ b/scripts/v.dissolve/tests/conftest.py @@ -10,7 +10,7 @@ def updates_as_transaction(table, cat_column, column, column_quote, cats, values): """Create SQL statement for categories and values for a given column""" - sql = ["BEGIN TRANSACTION"] + sql = ["BEGIN;"] if column_quote: quote = "'" else: @@ -20,7 +20,7 @@ def updates_as_transaction(table, cat_column, column, column_quote, cats, values f"UPDATE {table} SET {column} = {quote}{value}{quote} " f"WHERE {cat_column} = {cat};" ) - sql.append("END TRANSACTION") + sql.append("COMMIT;") return "\n".join(sql) diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index b5e030ad573..a5c19be064e 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -197,7 +197,7 @@ def sql_escape(text): def updates_to_sql(table, updates): """Create SQL from a list of dicts with column, value, where""" - sql = ["BEGIN TRANSACTION"] + sql = ["BEGIN;"] for update in updates: quote = quote_from_type(update.get("type", None)) value = update["value"] @@ -206,7 +206,7 @@ def updates_to_sql(table, updates): f"UPDATE {table} SET {update['column']} = {sql_value} " f"WHERE {update['where']};" ) - sql.append("END TRANSACTION") + sql.append("COMMIT;") return "\n".join(sql) From d37a92a6e136d0e73dfa183dff5d53b17863889a Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Mon, 22 Apr 2024 11:28:29 +0200 Subject: [PATCH 02/19] db.execute: start transaction before execute SQL and commit after --- db/db.execute/main.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/db/db.execute/main.c b/db/db.execute/main.c index 4f2a88244aa..bae75a94854 100644 --- a/db/db.execute/main.c +++ b/db/db.execute/main.c @@ -75,6 +75,10 @@ int main(int argc, char **argv) G_add_error_handler(error_handler, driver); if (parms.sql) { + ret = db_begin_transaction(driver); + if (ret != DB_OK) + G_fatal_error(_("Error while start transaction")); + /* parms.sql */ db_set_string(&stmt, parms.sql); ret = db_execute_immediate(driver, &stmt); @@ -90,11 +94,18 @@ int main(int argc, char **argv) db_get_string(&stmt)); } } + ret = db_commit_transaction(driver); + if (ret != DB_OK) + G_fatal_error(_("Error while commit transaction")); } else { /* parms.input */ while (get_stmt(fd, &stmt)) { if (stmt_is_empty(&stmt)) continue; + ret = db_begin_transaction(driver); + if (ret != DB_OK) + G_fatal_error(_("Error while start transaction")); + G_debug(3, "sql: %s", db_get_string(&stmt)); ret = db_execute_immediate(driver, &stmt); @@ -110,6 +121,9 @@ int main(int argc, char **argv) db_get_string(&stmt)); } } + ret = db_commit_transaction(driver); + if (ret != DB_OK) + G_fatal_error(_("Error while commit transaction")); } } From 861b7965f00cf07ca8b5f2e6666fe14baa6218ca Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Mon, 22 Apr 2024 11:28:58 +0200 Subject: [PATCH 03/19] Remove SQL BEGIN and COMMIT command transaction --- scripts/db.dropcolumn/db.dropcolumn.py | 2 -- scripts/v.db.addcolumn/v.db.addcolumn.py | 3 +-- scripts/v.db.dropcolumn/v.db.dropcolumn.py | 2 -- scripts/v.db.join/v.db.join.py | 4 ++-- scripts/v.db.univar/tests/conftest.py | 3 +-- scripts/v.dissolve/tests/conftest.py | 3 +-- scripts/v.dissolve/v.dissolve.py | 3 +-- 7 files changed, 6 insertions(+), 14 deletions(-) diff --git a/scripts/db.dropcolumn/db.dropcolumn.py b/scripts/db.dropcolumn/db.dropcolumn.py index e6f67716279..146983f447d 100755 --- a/scripts/db.dropcolumn/db.dropcolumn.py +++ b/scripts/db.dropcolumn/db.dropcolumn.py @@ -120,14 +120,12 @@ def main(): coltypes = ", ".join(coltypes) cmds = [ - "BEGIN;", "CREATE TEMPORARY TABLE ${table}_backup(${coldef})", "INSERT INTO ${table}_backup SELECT ${colnames} FROM ${table}", "DROP TABLE ${table}", "CREATE TABLE ${table}(${coldef})", "INSERT INTO ${table} SELECT ${colnames} FROM ${table}_backup", "DROP TABLE ${table}_backup", - "COMMIT;", ] tmpl = string.Template(";\n".join(cmds)) sql = tmpl.substitute(table=table, coldef=coltypes, colnames=colnames) diff --git a/scripts/v.db.addcolumn/v.db.addcolumn.py b/scripts/v.db.addcolumn/v.db.addcolumn.py index ed4ca712bb7..a3634150091 100755 --- a/scripts/v.db.addcolumn/v.db.addcolumn.py +++ b/scripts/v.db.addcolumn/v.db.addcolumn.py @@ -99,7 +99,7 @@ def main(): driver = f["driver"] column_existing = grass.vector_columns(map, int(layer)).keys() - add_str = "BEGIN;\n" + add_str = "" pattern = re.compile(r"\s+") for col in columns: if not col: @@ -120,7 +120,6 @@ def main(): continue grass.verbose(_("Adding column <{}> to the table").format(col_name)) add_str += f'ALTER TABLE {table} ADD COLUMN "{col_name}" {col_type};\n' - add_str += "COMMIT;" sql_file = grass.tempfile() rm_files.append(sql_file) cols_add_str = ",".join([col[0] for col in columns]) diff --git a/scripts/v.db.dropcolumn/v.db.dropcolumn.py b/scripts/v.db.dropcolumn/v.db.dropcolumn.py index a05a14c4bc8..1a47cd0b2e2 100755 --- a/scripts/v.db.dropcolumn/v.db.dropcolumn.py +++ b/scripts/v.db.dropcolumn/v.db.dropcolumn.py @@ -104,7 +104,6 @@ def main(): coltypes = ", ".join(coltypes) cmds = [ - "BEGIN;", "CREATE TEMPORARY TABLE ${table}_backup (${coldef})", "INSERT INTO ${table}_backup SELECT ${colnames} FROM ${table}", "DROP TABLE ${table}", @@ -112,7 +111,6 @@ def main(): "INSERT INTO ${table} SELECT ${colnames} FROM ${table}_backup", "CREATE UNIQUE INDEX ${table}_cat ON ${table} (${keycol} )", "DROP TABLE ${table}_backup", - "COMMIT;", ] tmpl = string.Template(";\n".join(cmds)) sql = tmpl.substitute( diff --git a/scripts/v.db.join/v.db.join.py b/scripts/v.db.join/v.db.join.py index 22696ab753f..39514ebdb91 100755 --- a/scripts/v.db.join/v.db.join.py +++ b/scripts/v.db.join/v.db.join.py @@ -225,8 +225,8 @@ def main(): f"{otable}.{ocolumn}={maptable}.{column});\n" ) update_str += cur_up_str - grass.debug(update_str, 1) - grass.verbose( + gs.debug(update_str, 1) + gs.verbose( _("Updating columns {columns} of vector map {map_name}...").format( columns=", ".join(cols_to_update.keys()), map_name=vector_map ) diff --git a/scripts/v.db.univar/tests/conftest.py b/scripts/v.db.univar/tests/conftest.py index 336aace4f19..7848167b6e8 100644 --- a/scripts/v.db.univar/tests/conftest.py +++ b/scripts/v.db.univar/tests/conftest.py @@ -9,10 +9,9 @@ def updates_as_transaction(table, cat_column, column, cats, values): """Create SQL statement for categories and values for a given column""" - sql = ["BEGIN;"] + sql = [] for cat, value in zip(cats, values): sql.append(f"UPDATE {table} SET {column} = {value} WHERE {cat_column} = {cat};") - sql.append("COMMIT;") return "\n".join(sql) diff --git a/scripts/v.dissolve/tests/conftest.py b/scripts/v.dissolve/tests/conftest.py index b9956857e1d..6a272cd5b41 100644 --- a/scripts/v.dissolve/tests/conftest.py +++ b/scripts/v.dissolve/tests/conftest.py @@ -10,7 +10,7 @@ def updates_as_transaction(table, cat_column, column, column_quote, cats, values): """Create SQL statement for categories and values for a given column""" - sql = ["BEGIN;"] + sql = [] if column_quote: quote = "'" else: @@ -20,7 +20,6 @@ def updates_as_transaction(table, cat_column, column, column_quote, cats, values f"UPDATE {table} SET {column} = {quote}{value}{quote} " f"WHERE {cat_column} = {cat};" ) - sql.append("COMMIT;") return "\n".join(sql) diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index a5c19be064e..206c35efa9b 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -197,7 +197,7 @@ def sql_escape(text): def updates_to_sql(table, updates): """Create SQL from a list of dicts with column, value, where""" - sql = ["BEGIN;"] + sql = [] for update in updates: quote = quote_from_type(update.get("type", None)) value = update["value"] @@ -206,7 +206,6 @@ def updates_to_sql(table, updates): f"UPDATE {table} SET {update['column']} = {sql_value} " f"WHERE {update['where']};" ) - sql.append("COMMIT;") return "\n".join(sql) From bce1cf3b8a221419b05b24e25d71b82897b21bcb Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Mon, 22 Apr 2024 12:30:21 +0200 Subject: [PATCH 04/19] Revert "db.execute: start transaction before execute SQL and commit after" This reverts commit 06bbc2353ebfae69f8fec9bb5e620b7d685d99be. --- db/db.execute/main.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/db/db.execute/main.c b/db/db.execute/main.c index bae75a94854..4f2a88244aa 100644 --- a/db/db.execute/main.c +++ b/db/db.execute/main.c @@ -75,10 +75,6 @@ int main(int argc, char **argv) G_add_error_handler(error_handler, driver); if (parms.sql) { - ret = db_begin_transaction(driver); - if (ret != DB_OK) - G_fatal_error(_("Error while start transaction")); - /* parms.sql */ db_set_string(&stmt, parms.sql); ret = db_execute_immediate(driver, &stmt); @@ -94,18 +90,11 @@ int main(int argc, char **argv) db_get_string(&stmt)); } } - ret = db_commit_transaction(driver); - if (ret != DB_OK) - G_fatal_error(_("Error while commit transaction")); } else { /* parms.input */ while (get_stmt(fd, &stmt)) { if (stmt_is_empty(&stmt)) continue; - ret = db_begin_transaction(driver); - if (ret != DB_OK) - G_fatal_error(_("Error while start transaction")); - G_debug(3, "sql: %s", db_get_string(&stmt)); ret = db_execute_immediate(driver, &stmt); @@ -121,9 +110,6 @@ int main(int argc, char **argv) db_get_string(&stmt)); } } - ret = db_commit_transaction(driver); - if (ret != DB_OK) - G_fatal_error(_("Error while commit transaction")); } } From a6ba7d20a8e3db825c712d94365abf4b10cbc408 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Mon, 22 Apr 2024 17:24:32 +0200 Subject: [PATCH 05/19] python/grass/script: refactor py db_begin_transaction(), db_commit_transaction() func To use C API db_begin_transaction(), db_commit_transaction(). Used by modules: - db.dropcolumn - v.db.addcolumn - v.db.dropcolumn - v.db.join.py - v.db.univar - v.dissolve - v.rast.stats --- python/grass/script/db.py | 84 ++++++++++++++++++---- scripts/db.dropcolumn/db.dropcolumn.py | 4 ++ scripts/v.db.addcolumn/v.db.addcolumn.py | 4 ++ scripts/v.db.dropcolumn/v.db.dropcolumn.py | 4 ++ scripts/v.db.join/v.db.join.py | 3 + scripts/v.db.univar/tests/conftest.py | 9 +++ scripts/v.dissolve/tests/conftest.py | 12 +++- scripts/v.dissolve/v.dissolve.py | 13 +++- scripts/v.rast.stats/v.rast.stats.py | 12 +++- 9 files changed, 128 insertions(+), 17 deletions(-) diff --git a/python/grass/script/db.py b/python/grass/script/db.py index abb7ad1764b..7ce947399f2 100644 --- a/python/grass/script/db.py +++ b/python/grass/script/db.py @@ -20,6 +20,9 @@ """ import os + +from ctypes import byref + from .core import ( run_command, parse_command, @@ -233,23 +236,80 @@ def db_table_in_vector(table, mapset=".", env=None): return None -def db_begin_transaction(driver): +def db_begin_transaction(driver_name, database): """Begin transaction. - :return: SQL command as string + :param str driver_name: DB driver name + :param str database: database name """ - if driver in ("sqlite", "pg"): - return "BEGIN" - if driver == "mysql": - return "START TRANSACTION" - return "" + try: + from grass.lib.dbmi import ( + db_begin_transaction, + db_start_driver_open_database, + DB_OK, + ) + from grass.lib.gis import G_gisinit + from grass.lib.vector import Map_info, Vect_subst_var + except (ImportError, OSError, TypeError) as e: + fatal(_("Unable to import C functions: {e}").format(e)) + + G_gisinit("") + map = Map_info() + + driver = db_start_driver_open_database( + driver_name, Vect_subst_var(database, byref(map)) + ) + if not driver: + fatal( + _("Unable to open database <{db}> by driver <{driver}>.").format( + db=database, driver=driver_name + ) + ) + + ret = db_begin_transaction(driver) + if ret != DB_OK: + fatal( + _( + "Error while start database <{db}> transaction" " by driver <{driver}>." + ).format(db=database, driver=driver_name) + ) -def db_commit_transaction(driver): +def db_commit_transaction(driver_name, database): """Commit transaction. - :return: SQL command as string + :param str driver_name: DB driver name + :param str database: database name """ - if driver in ("sqlite", "pg", "mysql"): - return "COMMIT" - return "" + try: + from grass.lib.dbmi import ( + db_commit_transaction, + db_start_driver_open_database, + DB_OK, + ) + from grass.lib.gis import G_gisinit + from grass.lib.vector import Map_info, Vect_subst_var + except (ImportError, OSError, TypeError) as e: + fatal(_("Unable to import C functions: {e}").format(e)) + + G_gisinit("") + map = Map_info() + + driver = db_start_driver_open_database( + driver_name, Vect_subst_var(database, byref(map)) + ) + if not driver: + fatal( + _("Unable to commit database <{db}> by driver <{driver}>.").format( + db=database, driver=driver_name + ) + ) + + ret = db_commit_transaction(driver) + if ret != DB_OK: + fatal( + _( + "Error while commit database <{db}> transaction" + " by driver <{driver}>." + ).format(db=database, driver=driver_name) + ) diff --git a/scripts/db.dropcolumn/db.dropcolumn.py b/scripts/db.dropcolumn/db.dropcolumn.py index 146983f447d..f5f27f8b6a4 100755 --- a/scripts/db.dropcolumn/db.dropcolumn.py +++ b/scripts/db.dropcolumn/db.dropcolumn.py @@ -50,6 +50,8 @@ def main(): + from grass.script.db import db_begin_transaction, db_commit_transaction + table = options["table"] column = options["column"] database = options["database"] @@ -133,9 +135,11 @@ def main(): sql = "ALTER TABLE %s DROP COLUMN %s" % (table, column) try: + db_begin_transaction(driver_name=driver, database=database) gscript.write_command( "db.execute", input="-", database=database, driver=driver, stdin=sql ) + db_commit_transaction(driver_name=driver, database=database) except CalledModuleError: gscript.fatal(_("Cannot continue (problem deleting column)")) diff --git a/scripts/v.db.addcolumn/v.db.addcolumn.py b/scripts/v.db.addcolumn/v.db.addcolumn.py index a3634150091..6795c96f21d 100755 --- a/scripts/v.db.addcolumn/v.db.addcolumn.py +++ b/scripts/v.db.addcolumn/v.db.addcolumn.py @@ -64,6 +64,8 @@ def cleanup(): def main(): + from grass.script.db import db_begin_transaction, db_commit_transaction + global rm_files map = options["map"] layer = options["layer"] @@ -126,12 +128,14 @@ def main(): with open(sql_file, "w") as write_file: write_file.write(add_str) try: + db_begin_transaction(driver_name=driver, database=database) grass.run_command( "db.execute", input=sql_file, database=database, driver=driver, ) + db_commit_transaction(driver_name=driver, database=database) except CalledModuleError: grass.fatal(_("Error adding columns {}").format(cols_add_str)) # write cmd history: diff --git a/scripts/v.db.dropcolumn/v.db.dropcolumn.py b/scripts/v.db.dropcolumn/v.db.dropcolumn.py index 1a47cd0b2e2..1a470076d40 100755 --- a/scripts/v.db.dropcolumn/v.db.dropcolumn.py +++ b/scripts/v.db.dropcolumn/v.db.dropcolumn.py @@ -43,6 +43,8 @@ def main(): + from grass.script.db import db_begin_transaction, db_commit_transaction + map = options["map"] layer = options["layer"] columns = options["columns"].split(",") @@ -120,9 +122,11 @@ def main(): sql = f'ALTER TABLE {table} DROP COLUMN "{column}"' try: + db_begin_transaction(driver_name=driver, database=database) grass.write_command( "db.execute", input="-", database=database, driver=driver, stdin=sql ) + db_commit_transaction(driver_name=driver, database=database) except CalledModuleError: grass.fatal(_("Deleting column failed")) diff --git a/scripts/v.db.join/v.db.join.py b/scripts/v.db.join/v.db.join.py index 39514ebdb91..3ca0d1f0d70 100755 --- a/scripts/v.db.join/v.db.join.py +++ b/scripts/v.db.join/v.db.join.py @@ -87,6 +87,8 @@ def cleanup(): def main(): + from grass.script.db import db_begin_transaction, db_commit_transaction + global rm_files # Include mapset into the name, so we avoid multiple messages about # found in more mapsets. The following generates an error message, while the code @@ -242,6 +244,7 @@ def main(): database=database, driver=driver, ) + db_commit_transaction(driver_name=driver, database=database) except CalledModuleError: gs.fatal(_("Error filling columns {}").format(cols_to_update)) diff --git a/scripts/v.db.univar/tests/conftest.py b/scripts/v.db.univar/tests/conftest.py index 7848167b6e8..f484adf7ea2 100644 --- a/scripts/v.db.univar/tests/conftest.py +++ b/scripts/v.db.univar/tests/conftest.py @@ -5,6 +5,7 @@ import pytest import grass.script as gs +from grass.script.db import db_begin_transaction, db_commit_transaction def updates_as_transaction(table, cat_column, column, cats, values): @@ -29,9 +30,17 @@ def value_update_by_category(map_name, layer, column_name, cats, values): cats=cats, values=values, ) + db_begin_transaction( + driver_name=db_info["driver"], + database=db_info["database"], + ) gs.write_command( "db.execute", input="-", database=database, driver=driver, stdin=sql ) + db_commit_transaction( + driver_name=db_info["driver"], + database=db_info["database"], + ) @pytest.fixture(scope="module") diff --git a/scripts/v.dissolve/tests/conftest.py b/scripts/v.dissolve/tests/conftest.py index 6a272cd5b41..f0ec816dfe4 100644 --- a/scripts/v.dissolve/tests/conftest.py +++ b/scripts/v.dissolve/tests/conftest.py @@ -6,9 +6,17 @@ import grass.script as gs import grass.script.setup as grass_setup +from grass.script.db import db_begin_transaction, db_commit_transaction -def updates_as_transaction(table, cat_column, column, column_quote, cats, values): +def updates_as_transaction( + table, + cat_column, + column, + column_quote, + cats, + values, +): """Create SQL statement for categories and values for a given column""" sql = [] if column_quote: @@ -40,9 +48,11 @@ def value_update_by_category(map_name, layer, column_name, cats, values): cats=cats, values=values, ) + db_begin_transaction(driver_name=driver, database=database) gs.write_command( "db.execute", input="-", database=database, driver=driver, stdin=sql ) + db_commit_transaction(driver_name=driver, database=database) @pytest.fixture(scope="module") diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index 206c35efa9b..336453f3bc3 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -83,7 +83,6 @@ import grass.script as gs from grass.exceptions import CalledModuleError - # Methods supported by v.db.univar by default. UNIVAR_METHODS = [ "n", @@ -211,6 +210,8 @@ def updates_to_sql(table, updates): def update_columns(output_name, output_layer, updates, add_columns): """Update attribute values based on a list of updates""" + from grass.script.db import db_begin_transaction, db_commit_transaction + if add_columns: gs.run_command( "v.db.addcolumn", @@ -220,6 +221,10 @@ def update_columns(output_name, output_layer, updates, add_columns): ) db_info = gs.vector_db(output_name)[int(output_layer)] sql = updates_to_sql(table=db_info["table"], updates=updates) + db_begin_transaction( + driver_name=db_info["driver"], + database=db_info["database"], + ) gs.write_command( "db.execute", input="-", @@ -227,6 +232,10 @@ def update_columns(output_name, output_layer, updates, add_columns): driver=db_info["driver"], stdin=sql, ) + db_commit_transaction( + driver_name=db_info["driver"], + database=db_info["database"], + ) def column_value_to_where(column, value, *, quote): @@ -548,7 +557,6 @@ def option_as_list(options, name): def main(): """Run the dissolve operation based on command line parameters""" - options, unused_flags = gs.parser() input_vector = options["input"] output = options["output"] layer = options["layer"] @@ -683,4 +691,5 @@ def main(): if __name__ == "__main__": + options, unused_flags = gs.parser() main() diff --git a/scripts/v.rast.stats/v.rast.stats.py b/scripts/v.rast.stats/v.rast.stats.py index c5a288a749d..0021c110794 100644 --- a/scripts/v.rast.stats/v.rast.stats.py +++ b/scripts/v.rast.stats/v.rast.stats.py @@ -93,6 +93,8 @@ def cleanup(): def main(): + from grass.script.db import db_begin_transaction, db_commit_transaction + global tmp, sqltmp, tmpname, nuldev, vector, rastertmp rastertmp = False # setup temporary files @@ -234,9 +236,17 @@ def main(): grass.message(_("Updating the database ...")) exitcode = 0 try: + db_begin_transaction( + driver_name=fi["driver"], + database=fi["database"], + ) grass.run_command( "db.execute", input=sqltmp, database=fi["database"], driver=fi["driver"] ) + db_commit_transaction( + driver_name=fi["driver"], + database=fi["database"], + ) grass.verbose( _( "Statistics calculated from raster map <{raster}>" @@ -472,7 +482,6 @@ def perform_stats( first_line = 1 - f.write("{0}\n".format(grass.db_begin_transaction(fi["driver"]))) for line in p.stdout: if first_line: first_line = 0 @@ -499,7 +508,6 @@ def perform_stats( f.write(" %s=%s" % (colname, value)) f.write(" WHERE %s=%s;\n" % (fi["key"], vars[0])) - f.write("{0}\n".format(grass.db_commit_transaction(fi["driver"]))) p.wait() From 47f799d3ab625b9bae4f2a033435d5871c07dc72 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Mon, 22 Apr 2024 17:55:13 +0200 Subject: [PATCH 06/19] Concatenate trans string --- python/grass/script/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/grass/script/db.py b/python/grass/script/db.py index 7ce947399f2..0765d1c1824 100644 --- a/python/grass/script/db.py +++ b/python/grass/script/db.py @@ -270,7 +270,7 @@ def db_begin_transaction(driver_name, database): if ret != DB_OK: fatal( _( - "Error while start database <{db}> transaction" " by driver <{driver}>." + "Error while start database <{db}> transaction by driver <{driver}>." ).format(db=database, driver=driver_name) ) From 68a04616d106a35dc51433bb34c93659cf69c105 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Mon, 22 Apr 2024 21:05:34 +0200 Subject: [PATCH 07/19] Exchange dbDriver* pointer between db_begin_transaction() and db_commit_transaction() and release opened driver/db connection --- python/grass/script/db.py | 31 +++++++++------------- scripts/db.dropcolumn/db.dropcolumn.py | 8 ++++-- scripts/v.db.addcolumn/v.db.addcolumn.py | 8 ++++-- scripts/v.db.dropcolumn/v.db.dropcolumn.py | 8 ++++-- scripts/v.db.join/v.db.join.py | 8 ++++-- scripts/v.db.univar/tests/conftest.py | 3 ++- scripts/v.dissolve/tests/conftest.py | 8 ++++-- scripts/v.dissolve/v.dissolve.py | 3 ++- scripts/v.rast.stats/v.rast.stats.py | 3 ++- 9 files changed, 49 insertions(+), 31 deletions(-) diff --git a/python/grass/script/db.py b/python/grass/script/db.py index 0765d1c1824..17facc008b0 100644 --- a/python/grass/script/db.py +++ b/python/grass/script/db.py @@ -241,11 +241,15 @@ def db_begin_transaction(driver_name, database): :param str driver_name: DB driver name :param str database: database name + + :return driver: opened driver/database connection + :rtype dbDriver* pointer """ try: from grass.lib.dbmi import ( db_begin_transaction, db_start_driver_open_database, + db_close_database_shutdown_driver, DB_OK, ) from grass.lib.gis import G_gisinit @@ -268,48 +272,39 @@ def db_begin_transaction(driver_name, database): ret = db_begin_transaction(driver) if ret != DB_OK: + db_close_database_shutdown_driver(driver) fatal( _( "Error while start database <{db}> transaction by driver <{driver}>." ).format(db=database, driver=driver_name) ) + return driver -def db_commit_transaction(driver_name, database): +def db_commit_transaction(driver_name, database, pdriver): """Commit transaction. :param str driver_name: DB driver name :param str database: database name + :param dbDriver* pointer pdriver: opened driver/database connection + pointer """ try: from grass.lib.dbmi import ( db_commit_transaction, - db_start_driver_open_database, + db_close_database_shutdown_driver, DB_OK, ) - from grass.lib.gis import G_gisinit - from grass.lib.vector import Map_info, Vect_subst_var except (ImportError, OSError, TypeError) as e: fatal(_("Unable to import C functions: {e}").format(e)) - G_gisinit("") - map = Map_info() - - driver = db_start_driver_open_database( - driver_name, Vect_subst_var(database, byref(map)) - ) - if not driver: - fatal( - _("Unable to commit database <{db}> by driver <{driver}>.").format( - db=database, driver=driver_name - ) - ) - - ret = db_commit_transaction(driver) + ret = db_commit_transaction(pdriver) if ret != DB_OK: + db_close_database_shutdown_driver(pdriver) fatal( _( "Error while commit database <{db}> transaction" " by driver <{driver}>." ).format(db=database, driver=driver_name) ) + db_close_database_shutdown_driver(pdriver) diff --git a/scripts/db.dropcolumn/db.dropcolumn.py b/scripts/db.dropcolumn/db.dropcolumn.py index f5f27f8b6a4..5dc198c4e6c 100755 --- a/scripts/db.dropcolumn/db.dropcolumn.py +++ b/scripts/db.dropcolumn/db.dropcolumn.py @@ -135,11 +135,15 @@ def main(): sql = "ALTER TABLE %s DROP COLUMN %s" % (table, column) try: - db_begin_transaction(driver_name=driver, database=database) + pdriver = db_begin_transaction(driver_name=driver, database=database) gscript.write_command( "db.execute", input="-", database=database, driver=driver, stdin=sql ) - db_commit_transaction(driver_name=driver, database=database) + db_commit_transaction( + driver_name=driver, + database=database, + pdriver=pdriver, + ) except CalledModuleError: gscript.fatal(_("Cannot continue (problem deleting column)")) diff --git a/scripts/v.db.addcolumn/v.db.addcolumn.py b/scripts/v.db.addcolumn/v.db.addcolumn.py index 6795c96f21d..7e5f3798207 100755 --- a/scripts/v.db.addcolumn/v.db.addcolumn.py +++ b/scripts/v.db.addcolumn/v.db.addcolumn.py @@ -128,14 +128,18 @@ def main(): with open(sql_file, "w") as write_file: write_file.write(add_str) try: - db_begin_transaction(driver_name=driver, database=database) + pdriver = db_begin_transaction(driver_name=driver, database=database) grass.run_command( "db.execute", input=sql_file, database=database, driver=driver, ) - db_commit_transaction(driver_name=driver, database=database) + db_commit_transaction( + driver_name=driver, + database=database, + pdriver=pdriver, + ) except CalledModuleError: grass.fatal(_("Error adding columns {}").format(cols_add_str)) # write cmd history: diff --git a/scripts/v.db.dropcolumn/v.db.dropcolumn.py b/scripts/v.db.dropcolumn/v.db.dropcolumn.py index 1a470076d40..8df928e92ae 100755 --- a/scripts/v.db.dropcolumn/v.db.dropcolumn.py +++ b/scripts/v.db.dropcolumn/v.db.dropcolumn.py @@ -122,11 +122,15 @@ def main(): sql = f'ALTER TABLE {table} DROP COLUMN "{column}"' try: - db_begin_transaction(driver_name=driver, database=database) + pdriver = db_begin_transaction(driver_name=driver, database=database) grass.write_command( "db.execute", input="-", database=database, driver=driver, stdin=sql ) - db_commit_transaction(driver_name=driver, database=database) + db_commit_transaction( + driver_name=driver, + database=database, + pdriver=pdriver, + ) except CalledModuleError: grass.fatal(_("Deleting column failed")) diff --git a/scripts/v.db.join/v.db.join.py b/scripts/v.db.join/v.db.join.py index 3ca0d1f0d70..1a28cb6e371 100755 --- a/scripts/v.db.join/v.db.join.py +++ b/scripts/v.db.join/v.db.join.py @@ -238,13 +238,17 @@ def main(): sql_file.write_text(update_str, encoding="UTF8") try: - gs.run_command( + grass.run_command( "db.execute", input=str(sql_file), database=database, driver=driver, ) - db_commit_transaction(driver_name=driver, database=database) + db_commit_transaction( + driver_name=driver, + database=database, + pdriver=pdriver, + ) except CalledModuleError: gs.fatal(_("Error filling columns {}").format(cols_to_update)) diff --git a/scripts/v.db.univar/tests/conftest.py b/scripts/v.db.univar/tests/conftest.py index f484adf7ea2..2d1b2a1d1b0 100644 --- a/scripts/v.db.univar/tests/conftest.py +++ b/scripts/v.db.univar/tests/conftest.py @@ -30,7 +30,7 @@ def value_update_by_category(map_name, layer, column_name, cats, values): cats=cats, values=values, ) - db_begin_transaction( + pdriver = db_begin_transaction( driver_name=db_info["driver"], database=db_info["database"], ) @@ -40,6 +40,7 @@ def value_update_by_category(map_name, layer, column_name, cats, values): db_commit_transaction( driver_name=db_info["driver"], database=db_info["database"], + pdriver=pdriver, ) diff --git a/scripts/v.dissolve/tests/conftest.py b/scripts/v.dissolve/tests/conftest.py index f0ec816dfe4..9789badae75 100644 --- a/scripts/v.dissolve/tests/conftest.py +++ b/scripts/v.dissolve/tests/conftest.py @@ -48,11 +48,15 @@ def value_update_by_category(map_name, layer, column_name, cats, values): cats=cats, values=values, ) - db_begin_transaction(driver_name=driver, database=database) + pdriver = db_begin_transaction(driver_name=driver, database=database) gs.write_command( "db.execute", input="-", database=database, driver=driver, stdin=sql ) - db_commit_transaction(driver_name=driver, database=database) + db_commit_transaction( + driver_name=driver, + database=database, + pdriver=pdriver, + ) @pytest.fixture(scope="module") diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index 336453f3bc3..c1ba04fcdcc 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -221,7 +221,7 @@ def update_columns(output_name, output_layer, updates, add_columns): ) db_info = gs.vector_db(output_name)[int(output_layer)] sql = updates_to_sql(table=db_info["table"], updates=updates) - db_begin_transaction( + pdriver = db_begin_transaction( driver_name=db_info["driver"], database=db_info["database"], ) @@ -235,6 +235,7 @@ def update_columns(output_name, output_layer, updates, add_columns): db_commit_transaction( driver_name=db_info["driver"], database=db_info["database"], + pdriver=pdriver, ) diff --git a/scripts/v.rast.stats/v.rast.stats.py b/scripts/v.rast.stats/v.rast.stats.py index 0021c110794..5f25d3878d1 100644 --- a/scripts/v.rast.stats/v.rast.stats.py +++ b/scripts/v.rast.stats/v.rast.stats.py @@ -236,7 +236,7 @@ def main(): grass.message(_("Updating the database ...")) exitcode = 0 try: - db_begin_transaction( + pdriver = db_begin_transaction( driver_name=fi["driver"], database=fi["database"], ) @@ -246,6 +246,7 @@ def main(): db_commit_transaction( driver_name=fi["driver"], database=fi["database"], + pdriver=pdriver, ) grass.verbose( _( From 95db46eb27819ffb89832ccd16debe6e8b77d2d5 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Tue, 23 Apr 2024 06:26:01 +0200 Subject: [PATCH 08/19] Add/use db_execute() func for reusing opened DB driver/connection --- python/grass/script/db.py | 31 +++++++ scripts/db.dropcolumn/db.dropcolumn.py | 19 +++-- scripts/v.db.addcolumn/v.db.addcolumn.py | 40 ++------- scripts/v.db.join/v.db.join.py | 44 +++------- scripts/v.db.univar/tests/conftest.py | 42 +++++++--- scripts/v.dissolve/tests/conftest.py | 32 +++++-- scripts/v.dissolve/v.dissolve.py | 42 ++++++---- scripts/v.rast.stats/v.rast.stats.py | 101 ++++++++++++----------- 8 files changed, 190 insertions(+), 161 deletions(-) diff --git a/python/grass/script/db.py b/python/grass/script/db.py index 17facc008b0..0ff302bc829 100644 --- a/python/grass/script/db.py +++ b/python/grass/script/db.py @@ -308,3 +308,34 @@ def db_commit_transaction(driver_name, database, pdriver): ).format(db=database, driver=driver_name) ) db_close_database_shutdown_driver(pdriver) + + +def db_execute(pdriver, sql): + """Execute SQL + + :param dbDriver* pointer pdriver: opened driver/database connection + pointer + :param str sql: SQL command + """ + try: + from grass.lib.dbmi import ( + dbString, + db_close_database_shutdown_driver, + db_execute_immediate, + db_free_string, + db_get_string, + db_init_string, + db_set_string, + DB_OK, + ) + except (ImportError, OSError, TypeError) as e: + fatal(_("Unable to import C functions: {e}").format(e)) + + stmt = dbString() + db_init_string(byref(stmt)) + db_set_string(byref(stmt), sql) + if db_execute_immediate(pdriver, byref(stmt)) != DB_OK: + db_free_string(byref(sql)) + db_close_database_shutdown_driver(pdriver) + fatal(_("Error while executing SQL <{}>.").format(db_get_string(byref(sql)))) + db_free_string(byref(stmt)) diff --git a/scripts/db.dropcolumn/db.dropcolumn.py b/scripts/db.dropcolumn/db.dropcolumn.py index 5dc198c4e6c..d004700079f 100755 --- a/scripts/db.dropcolumn/db.dropcolumn.py +++ b/scripts/db.dropcolumn/db.dropcolumn.py @@ -50,7 +50,11 @@ def main(): - from grass.script.db import db_begin_transaction, db_commit_transaction + from grass.script.db import ( + db_begin_transaction, + db_commit_transaction, + db_execute, + ) table = options["table"] column = options["column"] @@ -105,10 +109,11 @@ def main(): driver=driver, ).split(".")[0:2] + sqls = [] if [int(i) for i in sqlite3_version] >= [int(i) for i in "3.35".split(".")]: - sql = "ALTER TABLE %s DROP COLUMN %s" % (table, column) if column == "cat": - sql = "DROP INDEX %s_%s; %s" % (table, column, sql) + sqls.append(f"DROP INDEX {table}_{column};") + sqls.append(f"ALTER TABLE {table} DROP COLUMN {column};") else: # for older sqlite3 versions, use old way to remove column colnames = [] @@ -131,14 +136,14 @@ def main(): ] tmpl = string.Template(";\n".join(cmds)) sql = tmpl.substitute(table=table, coldef=coltypes, colnames=colnames) + sqls.extend(sql.split("\n")) else: - sql = "ALTER TABLE %s DROP COLUMN %s" % (table, column) + sqls.append("ALTER TABLE {table} DROP COLUMN {column};") try: pdriver = db_begin_transaction(driver_name=driver, database=database) - gscript.write_command( - "db.execute", input="-", database=database, driver=driver, stdin=sql - ) + for sql in sqls: + db_execute(pdriver=pdriver, sql=sql) db_commit_transaction( driver_name=driver, database=database, diff --git a/scripts/v.db.addcolumn/v.db.addcolumn.py b/scripts/v.db.addcolumn/v.db.addcolumn.py index 7e5f3798207..d0cff4a2018 100755 --- a/scripts/v.db.addcolumn/v.db.addcolumn.py +++ b/scripts/v.db.addcolumn/v.db.addcolumn.py @@ -40,31 +40,18 @@ # % key_desc: name type # %end -import atexit -import os import re from grass.exceptions import CalledModuleError import grass.script as grass -rm_files = [] - - -def cleanup(): - for file in rm_files: - if os.path.isfile(file): - try: - os.remove(file) - except Exception as e: - grass.warning( - _("Unable to remove file {file}: {message}").format( - file=file, message=e - ) - ) - def main(): - from grass.script.db import db_begin_transaction, db_commit_transaction + from grass.script.db import ( + db_begin_transaction, + db_commit_transaction, + db_execute, + ) global rm_files map = options["map"] @@ -101,7 +88,7 @@ def main(): driver = f["driver"] column_existing = grass.vector_columns(map, int(layer)).keys() - add_str = "" + sqls = [] pattern = re.compile(r"\s+") for col in columns: if not col: @@ -121,20 +108,12 @@ def main(): ) continue grass.verbose(_("Adding column <{}> to the table").format(col_name)) - add_str += f'ALTER TABLE {table} ADD COLUMN "{col_name}" {col_type};\n' - sql_file = grass.tempfile() - rm_files.append(sql_file) + sqls.append(f'ALTER TABLE {table} ADD COLUMN "{col_name}" {col_type};') cols_add_str = ",".join([col[0] for col in columns]) - with open(sql_file, "w") as write_file: - write_file.write(add_str) try: pdriver = db_begin_transaction(driver_name=driver, database=database) - grass.run_command( - "db.execute", - input=sql_file, - database=database, - driver=driver, - ) + for sql in sqls: + db_execute(pdriver=pdriver, sql=sql) db_commit_transaction( driver_name=driver, database=database, @@ -148,5 +127,4 @@ def main(): if __name__ == "__main__": options, flags = grass.parser() - atexit.register(cleanup) main() diff --git a/scripts/v.db.join/v.db.join.py b/scripts/v.db.join/v.db.join.py index 1a28cb6e371..67a1e35f4a7 100755 --- a/scripts/v.db.join/v.db.join.py +++ b/scripts/v.db.join/v.db.join.py @@ -63,33 +63,19 @@ # % description: Columns to exclude from the other table # %end -import atexit import sys -from pathlib import Path - import grass.script as gs from grass.exceptions import CalledModuleError -rm_files = [] - - -def cleanup(): - for file_path in rm_files: - try: - file_path.unlink(missing_ok=True) - except Exception as e: - gs.warning( - _("Unable to remove file {file}: {message}").format( - file=file_path, message=e - ) - ) - def main(): - from grass.script.db import db_begin_transaction, db_commit_transaction + from grass.script.db import ( + db_begin_transaction, + db_commit_transaction, + db_execute, + ) - global rm_files # Include mapset into the name, so we avoid multiple messages about # found in more mapsets. The following generates an error message, while the code # above does not. However, the above checks that the map exists, so we don't @@ -220,30 +206,21 @@ def main(): ) ) + sqls = [] for col in cols_to_update: cur_up_str = ( f"UPDATE {maptable} SET {col} = (SELECT {col} FROM " f"{otable} WHERE " - f"{otable}.{ocolumn}={maptable}.{column});\n" + f"{otable}.{ocolumn}={maptable}.{column});" ) - update_str += cur_up_str - gs.debug(update_str, 1) - gs.verbose( + sqls.apend(cur_up_str) + grass.debug("\n".join(sqls), 1) + grass.verbose( _("Updating columns {columns} of vector map {map_name}...").format( columns=", ".join(cols_to_update.keys()), map_name=vector_map ) ) - sql_file = Path(gs.tempfile()) - rm_files.append(sql_file) - sql_file.write_text(update_str, encoding="UTF8") - try: - grass.run_command( - "db.execute", - input=str(sql_file), - database=database, - driver=driver, - ) db_commit_transaction( driver_name=driver, database=database, @@ -260,5 +237,4 @@ def main(): if __name__ == "__main__": options, flags = gs.parser() - atexit.register(cleanup) sys.exit(main()) diff --git a/scripts/v.db.univar/tests/conftest.py b/scripts/v.db.univar/tests/conftest.py index 2d1b2a1d1b0..34f65bedf56 100644 --- a/scripts/v.db.univar/tests/conftest.py +++ b/scripts/v.db.univar/tests/conftest.py @@ -5,15 +5,32 @@ import pytest import grass.script as gs -from grass.script.db import db_begin_transaction, db_commit_transaction +from grass.script.db import ( + db_begin_transaction, + db_commit_transaction, + db_execute, +) def updates_as_transaction(table, cat_column, column, cats, values): - """Create SQL statement for categories and values for a given column""" - sql = [] + """Create SQL statement for categories and values for a given column + + :param str table: DB table name + :param str cat_column: DB table cat column name + :param str column: DB table update column name + :param list cats: DB table cat column values + :param str column: DB table update column name + :param list values: DB table update column values + + :return sqls: SQLs + :rtype list + """ + sqls = [] for cat, value in zip(cats, values): - sql.append(f"UPDATE {table} SET {column} = {value} WHERE {cat_column} = {cat};") - return "\n".join(sql) + sqls.append( + f"UPDATE {table} SET {column} = {value} WHERE {cat_column} = {cat};" + ) + return sqls def value_update_by_category(map_name, layer, column_name, cats, values): @@ -23,7 +40,7 @@ def value_update_by_category(map_name, layer, column_name, cats, values): database = db_info["database"] driver = db_info["driver"] cat_column = "cat" - sql = updates_as_transaction( + sqls = updates_as_transaction( table=table, cat_column=cat_column, column=column_name, @@ -31,15 +48,14 @@ def value_update_by_category(map_name, layer, column_name, cats, values): values=values, ) pdriver = db_begin_transaction( - driver_name=db_info["driver"], - database=db_info["database"], - ) - gs.write_command( - "db.execute", input="-", database=database, driver=driver, stdin=sql + driver_name=driver, + database=database, ) + for sql in sqls: + db_execute(pdriver=pdriver, sql=sql) db_commit_transaction( - driver_name=db_info["driver"], - database=db_info["database"], + driver_name=driver, + database=database, pdriver=pdriver, ) diff --git a/scripts/v.dissolve/tests/conftest.py b/scripts/v.dissolve/tests/conftest.py index 9789badae75..57c0ebde419 100644 --- a/scripts/v.dissolve/tests/conftest.py +++ b/scripts/v.dissolve/tests/conftest.py @@ -6,7 +6,11 @@ import grass.script as gs import grass.script.setup as grass_setup -from grass.script.db import db_begin_transaction, db_commit_transaction +from grass.script.db import ( + db_begin_transaction, + db_commit_transaction, + db_execute, +) def updates_as_transaction( @@ -17,18 +21,29 @@ def updates_as_transaction( cats, values, ): - """Create SQL statement for categories and values for a given column""" - sql = [] + """Create SQL statement for categories and values for a given column + + :param str table: DB table name + :param str cat_column: DB table cat column name + :param str column: DB table update column name + :param str column_quote: DB table column quote + :param list cats: DB table cat column values + :param list values: DB table update column values + + :return sqls: SQLs + :rtype list + """ + sqls = [] if column_quote: quote = "'" else: quote = "" for cat, value in zip(cats, values): - sql.append( + sqls.append( f"UPDATE {table} SET {column} = {quote}{value}{quote} " f"WHERE {cat_column} = {cat};" ) - return "\n".join(sql) + return sqls def value_update_by_category(map_name, layer, column_name, cats, values): @@ -40,7 +55,7 @@ def value_update_by_category(map_name, layer, column_name, cats, values): cat_column = "cat" column_type = gs.vector_columns(map_name, layer)[column_name] column_quote = bool(column_type["type"] in ("CHARACTER", "TEXT")) - sql = updates_as_transaction( + sqls = updates_as_transaction( table=table, cat_column=cat_column, column=column_name, @@ -49,9 +64,8 @@ def value_update_by_category(map_name, layer, column_name, cats, values): values=values, ) pdriver = db_begin_transaction(driver_name=driver, database=database) - gs.write_command( - "db.execute", input="-", database=database, driver=driver, stdin=sql - ) + for sql in sqls: + db_execute(pdriver=pdriver, sql=sql) db_commit_transaction( driver_name=driver, database=database, diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index c1ba04fcdcc..bba6f04786c 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -195,22 +195,33 @@ def sql_escape(text): def updates_to_sql(table, updates): - """Create SQL from a list of dicts with column, value, where""" - sql = [] + """Create SQL from a list of dicts with column, value, where + + :param str table: DB table name + :param list updates: DB table updates + + :return sqls: SQLs + :rtype list + """ + sqls = [] for update in updates: quote = quote_from_type(update.get("type", None)) value = update["value"] sql_value = f"{quote}{sql_escape(value) if value else 'NULL'}{quote}" - sql.append( + sqls.append( f"UPDATE {table} SET {update['column']} = {sql_value} " f"WHERE {update['where']};" ) - return "\n".join(sql) + return sqls def update_columns(output_name, output_layer, updates, add_columns): """Update attribute values based on a list of updates""" - from grass.script.db import db_begin_transaction, db_commit_transaction + from grass.script.db import ( + db_begin_transaction, + db_commit_transaction, + db_execute, + ) if add_columns: gs.run_command( @@ -220,21 +231,18 @@ def update_columns(output_name, output_layer, updates, add_columns): columns=",".join(add_columns), ) db_info = gs.vector_db(output_name)[int(output_layer)] - sql = updates_to_sql(table=db_info["table"], updates=updates) + driver = db_info["driver"] + database = db_info["database"] + sqls = updates_to_sql(table=db_info["table"], updates=updates) pdriver = db_begin_transaction( - driver_name=db_info["driver"], - database=db_info["database"], - ) - gs.write_command( - "db.execute", - input="-", - database=db_info["database"], - driver=db_info["driver"], - stdin=sql, + driver_name=driver, + database=database, ) + for sql in sqls: + db_execute(pdriver=pdriver, sql=sql) db_commit_transaction( - driver_name=db_info["driver"], - database=db_info["database"], + driver_name=driver, + database=database, pdriver=pdriver, ) diff --git a/scripts/v.rast.stats/v.rast.stats.py b/scripts/v.rast.stats/v.rast.stats.py index 5f25d3878d1..55ccd4343d9 100644 --- a/scripts/v.rast.stats/v.rast.stats.py +++ b/scripts/v.rast.stats/v.rast.stats.py @@ -93,13 +93,14 @@ def cleanup(): def main(): - from grass.script.db import db_begin_transaction, db_commit_transaction + from grass.script.db import ( + db_begin_transaction, + db_commit_transaction, + db_execute, + ) global tmp, sqltmp, tmpname, nuldev, vector, rastertmp rastertmp = False - # setup temporary files - tmp = grass.tempfile() - sqltmp = tmp + ".sql" # we need a random name tmpname = grass.basename(tmp) @@ -217,11 +218,8 @@ def main(): vector, layer, percentile, colprefixes[i], basecols, dbfdriver, flags["c"] ) - # get rid of any earlier attempts - grass.try_remove(sqltmp) - # do the stats - perform_stats( + sqls = perform_stats( raster, percentile, fi, @@ -240,9 +238,8 @@ def main(): driver_name=fi["driver"], database=fi["database"], ) - grass.run_command( - "db.execute", input=sqltmp, database=fi["database"], driver=fi["driver"] - ) + for sql in sqls: + db_execute(pdriver=pdriver, sql=sql) db_commit_transaction( driver_name=fi["driver"], database=fi["database"], @@ -470,46 +467,50 @@ def perform_stats( colnames, extstat, ): - with open(sqltmp, "w") as f: - # do the stats - p = grass.pipe_command( - "r.univar", - flags="t" + extstat, - map=raster, - zones=rastertmp, - percentile=percentile, - sep=";", - ) + sqls = [] + + # do the stats + p = grass.pipe_command( + "r.univar", + flags="t" + extstat, + map=raster, + zones=rastertmp, + percentile=percentile, + sep=";", + ) + + first_line = 1 + + for line in p.stdout: + if first_line: + first_line = 0 + continue + + vars = decode(line).rstrip("\r\n").split(";") + + sql = "" + sql += "UPDATE {fi['table']} SET" + first_var = 1 + for colname in colnames: + variable = colname.replace("%s_" % colprefix, "", 1) + if dbfdriver: + variable = variables_dbf[variable] + i = variables[variable] + value = vars[i] + # convert nan, +nan, -nan, inf, +inf, -inf, Infinity, +Infinity, + # -Infinity to NULL + if value.lower().endswith("nan") or "inf" in value.lower(): + value = "NULL" + if not first_var: + sql += " , " + else: + first_var = 0 + sql += f" {colname}={value}" - first_line = 1 - - for line in p.stdout: - if first_line: - first_line = 0 - continue - - vars = decode(line).rstrip("\r\n").split(";") - - f.write("UPDATE %s SET" % fi["table"]) - first_var = 1 - for colname in colnames: - variable = colname.replace("%s_" % colprefix, "", 1) - if dbfdriver: - variable = variables_dbf[variable] - i = variables[variable] - value = vars[i] - # convert nan, +nan, -nan, inf, +inf, -inf, Infinity, +Infinity, - # -Infinity to NULL - if value.lower().endswith("nan") or "inf" in value.lower(): - value = "NULL" - if not first_var: - f.write(" , ") - else: - first_var = 0 - f.write(" %s=%s" % (colname, value)) - - f.write(" WHERE %s=%s;\n" % (fi["key"], vars[0])) - p.wait() + sql += " WHERE {fi['key']}={vars[0]};" + sqls.append(sql) + p.wait() + return sqls if __name__ == "__main__": From 8c59bf7972c526c3863d3d93d5d0f22a96403478 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Tue, 23 Apr 2024 07:53:38 +0200 Subject: [PATCH 09/19] Fix removed tmp var --- scripts/v.rast.stats/v.rast.stats.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/v.rast.stats/v.rast.stats.py b/scripts/v.rast.stats/v.rast.stats.py index 55ccd4343d9..4081f054fa9 100644 --- a/scripts/v.rast.stats/v.rast.stats.py +++ b/scripts/v.rast.stats/v.rast.stats.py @@ -101,6 +101,8 @@ def main(): global tmp, sqltmp, tmpname, nuldev, vector, rastertmp rastertmp = False + # setup temporary files + tmp = grass.tempfile() # we need a random name tmpname = grass.basename(tmp) From 795a57ca51e8ac5021077571e65c799ba15557b2 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Tue, 23 Apr 2024 08:56:19 +0200 Subject: [PATCH 10/19] Fix make/print/append SQL string --- python/grass/script/db.py | 5 ++--- scripts/db.dropcolumn/db.dropcolumn.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/python/grass/script/db.py b/python/grass/script/db.py index 0ff302bc829..0a34641c498 100644 --- a/python/grass/script/db.py +++ b/python/grass/script/db.py @@ -323,7 +323,6 @@ def db_execute(pdriver, sql): db_close_database_shutdown_driver, db_execute_immediate, db_free_string, - db_get_string, db_init_string, db_set_string, DB_OK, @@ -335,7 +334,7 @@ def db_execute(pdriver, sql): db_init_string(byref(stmt)) db_set_string(byref(stmt), sql) if db_execute_immediate(pdriver, byref(stmt)) != DB_OK: - db_free_string(byref(sql)) + db_free_string(byref(stmt)) db_close_database_shutdown_driver(pdriver) - fatal(_("Error while executing SQL <{}>.").format(db_get_string(byref(sql)))) + fatal(_("Error while executing SQL <{}>.").format(sql)) db_free_string(byref(stmt)) diff --git a/scripts/db.dropcolumn/db.dropcolumn.py b/scripts/db.dropcolumn/db.dropcolumn.py index d004700079f..595f79444f4 100755 --- a/scripts/db.dropcolumn/db.dropcolumn.py +++ b/scripts/db.dropcolumn/db.dropcolumn.py @@ -100,6 +100,7 @@ def main(): ) return 0 + sqls = [] if driver == "sqlite": sqlite3_version = gscript.read_command( "db.select", @@ -109,7 +110,6 @@ def main(): driver=driver, ).split(".")[0:2] - sqls = [] if [int(i) for i in sqlite3_version] >= [int(i) for i in "3.35".split(".")]: if column == "cat": sqls.append(f"DROP INDEX {table}_{column};") @@ -138,7 +138,7 @@ def main(): sql = tmpl.substitute(table=table, coldef=coltypes, colnames=colnames) sqls.extend(sql.split("\n")) else: - sqls.append("ALTER TABLE {table} DROP COLUMN {column};") + sqls.append(f"ALTER TABLE {table} DROP COLUMN {column};") try: pdriver = db_begin_transaction(driver_name=driver, database=database) From c52978b52d061c52a44407169012a092518431c5 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Tue, 23 Apr 2024 10:20:57 +0200 Subject: [PATCH 11/19] Fix substituting SQLite DB path // var --- python/grass/script/db.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python/grass/script/db.py b/python/grass/script/db.py index 0a34641c498..26661f5b4dc 100644 --- a/python/grass/script/db.py +++ b/python/grass/script/db.py @@ -24,6 +24,7 @@ from ctypes import byref from .core import ( + gisenv, run_command, parse_command, read_command, @@ -140,6 +141,15 @@ def db_connection(force=False, env=None): run_command("db.connect", flags="c", env=env) conn = parse_command("db.connect", flags="g", env=env) + if conn.get("driver") == "sqlite": + gis_env = gisenv() + conn["database"] = ( + conn["database"] + .replace("$GISDBASE", gis_env["GISDBASE"]) + .replace("$LOCATION_NAME", gis_env["LOCATION_NAME"]) + .replace("$MAPSET", gis_env["MAPSET"]) + ) + return conn From 4d296630c8be64d1d8e7755cea717afb60b8733a Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Tue, 23 Apr 2024 10:41:30 +0200 Subject: [PATCH 12/19] Check if conn var is not None --- python/grass/script/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/grass/script/db.py b/python/grass/script/db.py index 26661f5b4dc..e6a9940c095 100644 --- a/python/grass/script/db.py +++ b/python/grass/script/db.py @@ -141,7 +141,7 @@ def db_connection(force=False, env=None): run_command("db.connect", flags="c", env=env) conn = parse_command("db.connect", flags="g", env=env) - if conn.get("driver") == "sqlite": + if conn and conn.get("driver") == "sqlite": gis_env = gisenv() conn["database"] = ( conn["database"] From 7c9f7fde3135446423e97b2220d57a8292f9faf6 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Tue, 23 Apr 2024 11:10:36 +0200 Subject: [PATCH 13/19] Fix f-string syntax --- scripts/v.rast.stats/v.rast.stats.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/v.rast.stats/v.rast.stats.py b/scripts/v.rast.stats/v.rast.stats.py index 4081f054fa9..dc563661ca3 100644 --- a/scripts/v.rast.stats/v.rast.stats.py +++ b/scripts/v.rast.stats/v.rast.stats.py @@ -491,7 +491,7 @@ def perform_stats( vars = decode(line).rstrip("\r\n").split(";") sql = "" - sql += "UPDATE {fi['table']} SET" + sql += f"UPDATE {fi['table']} SET" first_var = 1 for colname in colnames: variable = colname.replace("%s_" % colprefix, "", 1) @@ -509,7 +509,7 @@ def perform_stats( first_var = 0 sql += f" {colname}={value}" - sql += " WHERE {fi['key']}={vars[0]};" + sql += f" WHERE {fi['key']}={vars[0]};" sqls.append(sql) p.wait() return sqls From eef718e56e70ea55101db7c1a23299b98d1b15b0 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Tue, 23 Apr 2024 14:24:41 +0200 Subject: [PATCH 14/19] Fix append() method name --- scripts/v.db.join/v.db.join.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/v.db.join/v.db.join.py b/scripts/v.db.join/v.db.join.py index 67a1e35f4a7..4d8e108e72b 100755 --- a/scripts/v.db.join/v.db.join.py +++ b/scripts/v.db.join/v.db.join.py @@ -213,9 +213,9 @@ def main(): f"{otable} WHERE " f"{otable}.{ocolumn}={maptable}.{column});" ) - sqls.apend(cur_up_str) - grass.debug("\n".join(sqls), 1) - grass.verbose( + sqls.append(cur_up_str) + gs.debug("".join(sqls), 1) + gs.verbose( _("Updating columns {columns} of vector map {map_name}...").format( columns=", ".join(cols_to_update.keys()), map_name=vector_map ) From cc72ad31e31ebc71f629499e9963fe279b94c793 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Thu, 6 Jun 2024 14:08:17 +0200 Subject: [PATCH 15/19] Refactor code to use DBHandler class instance --- python/grass/script/db.py | 217 ++++++++++++--------- scripts/db.dropcolumn/db.dropcolumn.py | 17 +- scripts/v.db.addcolumn/v.db.addcolumn.py | 17 +- scripts/v.db.dropcolumn/v.db.dropcolumn.py | 35 ++-- scripts/v.db.join/v.db.join.py | 14 +- scripts/v.db.univar/tests/conftest.py | 19 +- scripts/v.dissolve/tests/conftest.py | 16 +- scripts/v.dissolve/v.dissolve.py | 19 +- scripts/v.rast.stats/v.rast.stats.py | 20 +- 9 files changed, 162 insertions(+), 212 deletions(-) diff --git a/python/grass/script/db.py b/python/grass/script/db.py index e6a9940c095..930d665e077 100644 --- a/python/grass/script/db.py +++ b/python/grass/script/db.py @@ -246,105 +246,138 @@ def db_table_in_vector(table, mapset=".", env=None): return None -def db_begin_transaction(driver_name, database): - """Begin transaction. +class DBHandler: + """DB handler - :param str driver_name: DB driver name - :param str database: database name + Allow execute SQL command(s) in transaction mode. - :return driver: opened driver/database connection - :rtype dbDriver* pointer + Public methods: + + ::execute """ - try: - from grass.lib.dbmi import ( - db_begin_transaction, - db_start_driver_open_database, - db_close_database_shutdown_driver, - DB_OK, - ) - from grass.lib.gis import G_gisinit - from grass.lib.vector import Map_info, Vect_subst_var - except (ImportError, OSError, TypeError) as e: - fatal(_("Unable to import C functions: {e}").format(e)) - - G_gisinit("") - map = Map_info() - - driver = db_start_driver_open_database( - driver_name, Vect_subst_var(database, byref(map)) - ) - if not driver: - fatal( - _("Unable to open database <{db}> by driver <{driver}>.").format( - db=database, driver=driver_name + + def __init__(self, driver_name, database): + """Constructor + + :param str driver_name: DB driver name + :param str database: database name + """ + self._driver_name = driver_name + self._database = database + self._import_c_funcs() + + def _import_c_funcs(self): + """Import C functions""" + try: + from grass.lib.dbmi import ( + db_begin_transaction, + db_close_database_shutdown_driver, + db_commit_transaction, + db_execute_immediate, + db_free_string, + db_init_string, + db_set_string, + db_start_driver_open_database, + dbString, + DB_OK, + ) + from grass.lib.gis import G_gisinit + from grass.lib.vector import ( + Map_info, + Vect_subst_var, ) - ) - ret = db_begin_transaction(driver) - if ret != DB_OK: - db_close_database_shutdown_driver(driver) - fatal( - _( - "Error while start database <{db}> transaction by driver <{driver}>." - ).format(db=database, driver=driver_name) + self._c_funcs = { + "db_begin_transaction": db_begin_transaction, + "db_execute_immediate": db_execute_immediate, + "db_free_string": db_free_string, + "db_init_string": db_init_string, + "db_close_database_shutdown_driver": db_close_database_shutdown_driver, + "db_commit_transaction": db_commit_transaction, + "db_set_string": db_set_string, + "db_start_driver_open_database": db_start_driver_open_database, + "dbString": dbString, + "DB_OK": DB_OK, + "G_gisinit": G_gisinit, + "Map_info": Map_info, + "Vect_subst_var": Vect_subst_var, + } + except (ImportError, OSError, TypeError) as e: + fatal(_("Unable to import C functions: {e}").format(e)) + + def _init_driver(self): + """Init DB driver""" + map = self._c_funcs["Map_info"]() + self._pdriver = self._c_funcs["db_start_driver_open_database"]( + self._driver_name, + self._c_funcs["Vect_subst_var"](self._database, byref(map)), ) - return driver - - -def db_commit_transaction(driver_name, database, pdriver): - """Commit transaction. - - :param str driver_name: DB driver name - :param str database: database name - :param dbDriver* pointer pdriver: opened driver/database connection - pointer - """ - try: - from grass.lib.dbmi import ( - db_commit_transaction, - db_close_database_shutdown_driver, - DB_OK, - ) - except (ImportError, OSError, TypeError) as e: - fatal(_("Unable to import C functions: {e}").format(e)) + if not self._pdriver: + fatal( + _("Unable to open database <{db}> by driver <{driver}>.").format( + db=self._database, driver=self._driver_name + ) + ) - ret = db_commit_transaction(pdriver) - if ret != DB_OK: - db_close_database_shutdown_driver(pdriver) - fatal( - _( - "Error while commit database <{db}> transaction" - " by driver <{driver}>." - ).format(db=database, driver=driver_name) - ) - db_close_database_shutdown_driver(pdriver) + def _begin_transaction(self): + """Begin DB transaction.""" + self._c_funcs["G_gisinit"]("") + self._init_driver() -def db_execute(pdriver, sql): - """Execute SQL + ret = self._c_funcs["db_begin_transaction"](self._pdriver) + if ret != self._c_funcs["DB_OK"]: + self._c_funcs["db_close_database_shutdown_driver"](self._pdriver) + fatal( + _( + "Error while start database <{db}> transaction by driver <{driver}>." + ).format(db=self._database, driver=self._driver_name) + ) - :param dbDriver* pointer pdriver: opened driver/database connection - pointer - :param str sql: SQL command - """ - try: - from grass.lib.dbmi import ( - dbString, - db_close_database_shutdown_driver, - db_execute_immediate, - db_free_string, - db_init_string, - db_set_string, - DB_OK, - ) - except (ImportError, OSError, TypeError) as e: - fatal(_("Unable to import C functions: {e}").format(e)) - - stmt = dbString() - db_init_string(byref(stmt)) - db_set_string(byref(stmt), sql) - if db_execute_immediate(pdriver, byref(stmt)) != DB_OK: - db_free_string(byref(stmt)) - db_close_database_shutdown_driver(pdriver) - fatal(_("Error while executing SQL <{}>.").format(sql)) - db_free_string(byref(stmt)) + def _commit_transaction(self): + """Commit DB transaction.""" + ret = self._c_funcs["db_commit_transaction"](self._pdriver) + if ret != self._c_funcs["DB_OK"]: + self._c_funcs["db_close_database_shutdown_driver"](self._pdriver) + fatal( + _( + "Error while commit database <{db}> transaction" + " by driver <{driver}>." + ).format(db=self._database, driver=self._driver_name) + ) + self._c_funcs["db_close_database_shutdown_driver"](self._pdriver) + + def _execute(self, sql): + """Execute SQL + + :param str|list|tuple sql: SQL command string or list of SQLs + commands + """ + stmt = self._c_funcs["dbString"]() + self._c_funcs["db_init_string"](byref(stmt)) + self._c_funcs["db_set_string"](byref(stmt), sql) + if ( + self._c_funcs["db_execute_immediate"](self._pdriver, byref(stmt)) + != self._c_funcs["DB_OK"] + ): + self._c_funcs["db_free_string"](byref(stmt)) + self._c_funcs["db_close_database_shutdown_driver"](self._pdriver) + fatal(_("Error while executing SQL <{}>.").format(sql)) + self._c_funcs["db_free_string"](byref(stmt)) + + def execute(self, sql): + """Execute SQL + + :param str|list|tuple sql: SQL command string or list of SQLs + statement + """ + # Begin DB transaction + self._begin_transaction() + # Execute SQL string + if isinstance(sql, (list, tuple)): + for statement in sql: + self._execute(sql=statement) + else: + self._execute(sql) + # Commit DB transaction + self._commit_transaction() diff --git a/scripts/db.dropcolumn/db.dropcolumn.py b/scripts/db.dropcolumn/db.dropcolumn.py index 595f79444f4..b55b863322d 100755 --- a/scripts/db.dropcolumn/db.dropcolumn.py +++ b/scripts/db.dropcolumn/db.dropcolumn.py @@ -50,11 +50,7 @@ def main(): - from grass.script.db import ( - db_begin_transaction, - db_commit_transaction, - db_execute, - ) + from grass.script.db import DBHandler table = options["table"] column = options["column"] @@ -62,6 +58,8 @@ def main(): driver = options["driver"] force = flags["f"] + db_handler = DBHandler(driver_name=driver, database=database) + # check if DB parameters are set, and if not set them. gscript.run_command("db.connect", flags="c") @@ -141,14 +139,7 @@ def main(): sqls.append(f"ALTER TABLE {table} DROP COLUMN {column};") try: - pdriver = db_begin_transaction(driver_name=driver, database=database) - for sql in sqls: - db_execute(pdriver=pdriver, sql=sql) - db_commit_transaction( - driver_name=driver, - database=database, - pdriver=pdriver, - ) + db_handler.execute(sql=";".join(sqls)) except CalledModuleError: gscript.fatal(_("Cannot continue (problem deleting column)")) diff --git a/scripts/v.db.addcolumn/v.db.addcolumn.py b/scripts/v.db.addcolumn/v.db.addcolumn.py index d0cff4a2018..774bbfebfdd 100755 --- a/scripts/v.db.addcolumn/v.db.addcolumn.py +++ b/scripts/v.db.addcolumn/v.db.addcolumn.py @@ -47,11 +47,7 @@ def main(): - from grass.script.db import ( - db_begin_transaction, - db_commit_transaction, - db_execute, - ) + from grass.script.db import DBHandler global rm_files map = options["map"] @@ -88,6 +84,8 @@ def main(): driver = f["driver"] column_existing = grass.vector_columns(map, int(layer)).keys() + db_handler = DBHandler(driver_name=driver, database=database) + sqls = [] pattern = re.compile(r"\s+") for col in columns: @@ -111,14 +109,7 @@ def main(): sqls.append(f'ALTER TABLE {table} ADD COLUMN "{col_name}" {col_type};') cols_add_str = ",".join([col[0] for col in columns]) try: - pdriver = db_begin_transaction(driver_name=driver, database=database) - for sql in sqls: - db_execute(pdriver=pdriver, sql=sql) - db_commit_transaction( - driver_name=driver, - database=database, - pdriver=pdriver, - ) + db_handler.execute(sql=sqls) except CalledModuleError: grass.fatal(_("Error adding columns {}").format(cols_add_str)) # write cmd history: diff --git a/scripts/v.db.dropcolumn/v.db.dropcolumn.py b/scripts/v.db.dropcolumn/v.db.dropcolumn.py index 8df928e92ae..7172ae541da 100755 --- a/scripts/v.db.dropcolumn/v.db.dropcolumn.py +++ b/scripts/v.db.dropcolumn/v.db.dropcolumn.py @@ -37,13 +37,12 @@ # % required: yes # %end -import string import grass.script as grass from grass.exceptions import CalledModuleError def main(): - from grass.script.db import db_begin_transaction, db_commit_transaction + from grass.script.db import DBHandler map = options["map"] layer = options["layer"] @@ -62,6 +61,8 @@ def main(): database = f["database"] driver = f["driver"] + db_handler = DBHandler(driver_name=driver, database=database) + if not table: grass.fatal( _( @@ -105,32 +106,20 @@ def main(): colnames = ", ".join([f'"{col}"' for col in colnames]) coltypes = ", ".join(coltypes) - cmds = [ - "CREATE TEMPORARY TABLE ${table}_backup (${coldef})", - "INSERT INTO ${table}_backup SELECT ${colnames} FROM ${table}", - "DROP TABLE ${table}", - "CREATE TABLE ${table}(${coldef})", - "INSERT INTO ${table} SELECT ${colnames} FROM ${table}_backup", - "CREATE UNIQUE INDEX ${table}_cat ON ${table} (${keycol} )", - "DROP TABLE ${table}_backup", + sql = [ + f"CREATE TEMPORARY TABLE {table}_backup ({coltypes})", + f"INSERT INTO {table}_backup SELECT {colnames} FROM {table}", + f"DROP TABLE {table}", + f"CREATE TABLE {table}({coltypes})", + f"INSERT INTO {table} SELECT {colnames} FROM {table}_backup", + f"CREATE UNIQUE INDEX {table}_cat ON {table} ({keycol} )", + f"DROP TABLE {table}_backup", ] - tmpl = string.Template(";\n".join(cmds)) - sql = tmpl.substitute( - table=table, coldef=coltypes, colnames=colnames, keycol=keycol - ) else: sql = f'ALTER TABLE {table} DROP COLUMN "{column}"' try: - pdriver = db_begin_transaction(driver_name=driver, database=database) - grass.write_command( - "db.execute", input="-", database=database, driver=driver, stdin=sql - ) - db_commit_transaction( - driver_name=driver, - database=database, - pdriver=pdriver, - ) + db_handler.execute(sql=sql) except CalledModuleError: grass.fatal(_("Deleting column failed")) diff --git a/scripts/v.db.join/v.db.join.py b/scripts/v.db.join/v.db.join.py index 4d8e108e72b..4ebf6551b6b 100755 --- a/scripts/v.db.join/v.db.join.py +++ b/scripts/v.db.join/v.db.join.py @@ -70,11 +70,7 @@ def main(): - from grass.script.db import ( - db_begin_transaction, - db_commit_transaction, - db_execute, - ) + from grass.script.db import DBHandler # Include mapset into the name, so we avoid multiple messages about # found in more mapsets. The following generates an error message, while the code @@ -101,6 +97,8 @@ def main(): database = f["database"] driver = f["driver"] + db_handler = DBHandler(driver_name=driver, database=database) + if driver == "dbf": gs.fatal(_("JOIN is not supported for tables stored in DBF format")) @@ -221,11 +219,7 @@ def main(): ) ) try: - db_commit_transaction( - driver_name=driver, - database=database, - pdriver=pdriver, - ) + db_handler.execute(sql=sqls) except CalledModuleError: gs.fatal(_("Error filling columns {}").format(cols_to_update)) diff --git a/scripts/v.db.univar/tests/conftest.py b/scripts/v.db.univar/tests/conftest.py index 34f65bedf56..5e19e541f87 100644 --- a/scripts/v.db.univar/tests/conftest.py +++ b/scripts/v.db.univar/tests/conftest.py @@ -5,11 +5,7 @@ import pytest import grass.script as gs -from grass.script.db import ( - db_begin_transaction, - db_commit_transaction, - db_execute, -) +from grass.script.db import DBHandler def updates_as_transaction(table, cat_column, column, cats, values): @@ -47,17 +43,8 @@ def value_update_by_category(map_name, layer, column_name, cats, values): cats=cats, values=values, ) - pdriver = db_begin_transaction( - driver_name=driver, - database=database, - ) - for sql in sqls: - db_execute(pdriver=pdriver, sql=sql) - db_commit_transaction( - driver_name=driver, - database=database, - pdriver=pdriver, - ) + db_handler = DBHandler(driver_name=driver, database=database) + db_handler.execute(sql=sqls) @pytest.fixture(scope="module") diff --git a/scripts/v.dissolve/tests/conftest.py b/scripts/v.dissolve/tests/conftest.py index 57c0ebde419..f0abe4ef602 100644 --- a/scripts/v.dissolve/tests/conftest.py +++ b/scripts/v.dissolve/tests/conftest.py @@ -6,11 +6,7 @@ import grass.script as gs import grass.script.setup as grass_setup -from grass.script.db import ( - db_begin_transaction, - db_commit_transaction, - db_execute, -) +from grass.script.db import DBHandler def updates_as_transaction( @@ -63,14 +59,8 @@ def value_update_by_category(map_name, layer, column_name, cats, values): cats=cats, values=values, ) - pdriver = db_begin_transaction(driver_name=driver, database=database) - for sql in sqls: - db_execute(pdriver=pdriver, sql=sql) - db_commit_transaction( - driver_name=driver, - database=database, - pdriver=pdriver, - ) + db_handler = DBHandler(driver_name=driver, database=database) + db_handler.execute(sql=sqls) @pytest.fixture(scope="module") diff --git a/scripts/v.dissolve/v.dissolve.py b/scripts/v.dissolve/v.dissolve.py index bba6f04786c..ef47b7115da 100755 --- a/scripts/v.dissolve/v.dissolve.py +++ b/scripts/v.dissolve/v.dissolve.py @@ -217,11 +217,7 @@ def updates_to_sql(table, updates): def update_columns(output_name, output_layer, updates, add_columns): """Update attribute values based on a list of updates""" - from grass.script.db import ( - db_begin_transaction, - db_commit_transaction, - db_execute, - ) + from grass.script.db import DBHandler if add_columns: gs.run_command( @@ -234,17 +230,8 @@ def update_columns(output_name, output_layer, updates, add_columns): driver = db_info["driver"] database = db_info["database"] sqls = updates_to_sql(table=db_info["table"], updates=updates) - pdriver = db_begin_transaction( - driver_name=driver, - database=database, - ) - for sql in sqls: - db_execute(pdriver=pdriver, sql=sql) - db_commit_transaction( - driver_name=driver, - database=database, - pdriver=pdriver, - ) + db_handler = DBHandler(driver_name=driver, database=database) + db_handler.execute(sql=sqls) def column_value_to_where(column, value, *, quote): diff --git a/scripts/v.rast.stats/v.rast.stats.py b/scripts/v.rast.stats/v.rast.stats.py index dc563661ca3..d26e7aec919 100644 --- a/scripts/v.rast.stats/v.rast.stats.py +++ b/scripts/v.rast.stats/v.rast.stats.py @@ -93,11 +93,7 @@ def cleanup(): def main(): - from grass.script.db import ( - db_begin_transaction, - db_commit_transaction, - db_execute, - ) + from grass.script.db import DBHandler global tmp, sqltmp, tmpname, nuldev, vector, rastertmp rastertmp = False @@ -213,6 +209,8 @@ def main(): # calculate statistics: grass.message(_("Processing input data (%d categories)...") % number) + db_handler = DBHandler(driver_name=fi["driver"], database=fi["database"]) + for i in range(len(rasters)): raster = rasters[i] @@ -236,17 +234,7 @@ def main(): grass.message(_("Updating the database ...")) exitcode = 0 try: - pdriver = db_begin_transaction( - driver_name=fi["driver"], - database=fi["database"], - ) - for sql in sqls: - db_execute(pdriver=pdriver, sql=sql) - db_commit_transaction( - driver_name=fi["driver"], - database=fi["database"], - pdriver=pdriver, - ) + db_handler.execute(sql=sqls) grass.verbose( _( "Statistics calculated from raster map <{raster}>" From b073c21894a5e37d57d0c7727e4c6fff0aa1e0c6 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Thu, 6 Jun 2024 15:37:40 +0200 Subject: [PATCH 16/19] Fix Flake8 E501 line too long (89 > 88 characters) error --- python/grass/script/db.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/grass/script/db.py b/python/grass/script/db.py index 930d665e077..9d466a30837 100644 --- a/python/grass/script/db.py +++ b/python/grass/script/db.py @@ -330,7 +330,8 @@ def _begin_transaction(self): self._c_funcs["db_close_database_shutdown_driver"](self._pdriver) fatal( _( - "Error while start database <{db}> transaction by driver <{driver}>." + "Error while start database <{db}> transaction by" + " driver <{driver}>." ).format(db=self._database, driver=self._driver_name) ) From 072d0be7622dc177d6e5c993017577060be80923 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Mon, 10 Jun 2024 22:03:17 +0200 Subject: [PATCH 17/19] Incorporate suggestions --- python/grass/script/db.py | 92 ++++++++++++---------- scripts/db.dropcolumn/db.dropcolumn.py | 5 +- scripts/v.db.addcolumn/v.db.addcolumn.py | 6 +- scripts/v.db.dropcolumn/v.db.dropcolumn.py | 3 +- scripts/v.db.join/v.db.join.py | 3 +- scripts/v.rast.stats/v.rast.stats.py | 6 +- 6 files changed, 59 insertions(+), 56 deletions(-) diff --git a/python/grass/script/db.py b/python/grass/script/db.py index 9d466a30837..ca119549281 100644 --- a/python/grass/script/db.py +++ b/python/grass/script/db.py @@ -264,10 +264,10 @@ def __init__(self, driver_name, database): """ self._driver_name = driver_name self._database = database - self._import_c_funcs() + self._import_c_interface() - def _import_c_funcs(self): - """Import C functions""" + def _import_c_interface(self): + """Import C interface""" try: from grass.lib.dbmi import ( db_begin_transaction, @@ -286,31 +286,35 @@ def _import_c_funcs(self): Map_info, Vect_subst_var, ) - - self._c_funcs = { - "db_begin_transaction": db_begin_transaction, - "db_execute_immediate": db_execute_immediate, - "db_free_string": db_free_string, - "db_init_string": db_init_string, - "db_close_database_shutdown_driver": db_close_database_shutdown_driver, - "db_commit_transaction": db_commit_transaction, - "db_set_string": db_set_string, - "db_start_driver_open_database": db_start_driver_open_database, - "dbString": dbString, - "DB_OK": DB_OK, - "G_gisinit": G_gisinit, - "Map_info": Map_info, - "Vect_subst_var": Vect_subst_var, - } except (ImportError, OSError, TypeError) as e: fatal(_("Unable to import C functions: {e}").format(e)) + class CInterface: + def __init__(self): + self.db_begin_transaction = db_begin_transaction + self.db_execute_immediate = db_execute_immediate + self.db_free_string = db_free_string + self.db_init_string = db_init_string + self.db_close_database_shutdown_driver = ( + db_close_database_shutdown_driver + ) + self.db_commit_transaction = db_commit_transaction + self.db_set_string = db_set_string + self.db_start_driver_open_database = db_start_driver_open_database + self.dbString = dbString + self.DB_OK = DB_OK + self.G_gisinit = G_gisinit + self.Map_info = Map_info + self.Vect_subst_var = Vect_subst_var + + self._c_interface = CInterface() + def _init_driver(self): """Init DB driver""" - map = self._c_funcs["Map_info"]() - self._pdriver = self._c_funcs["db_start_driver_open_database"]( + map = self._c_interface.Map_info() + self._pdriver = self._c_interface.db_start_driver_open_database( self._driver_name, - self._c_funcs["Vect_subst_var"](self._database, byref(map)), + self._c_interface.Vect_subst_var(self._database, byref(map)), ) if not self._pdriver: fatal( @@ -321,32 +325,27 @@ def _init_driver(self): def _begin_transaction(self): """Begin DB transaction.""" - self._c_funcs["G_gisinit"]("") - - self._init_driver() - - ret = self._c_funcs["db_begin_transaction"](self._pdriver) - if ret != self._c_funcs["DB_OK"]: - self._c_funcs["db_close_database_shutdown_driver"](self._pdriver) + ret = self._c_interface.db_begin_transaction(self._pdriver) + if ret != self._c_interface.DB_OK: + self._shutdown_driver() fatal( _( - "Error while start database <{db}> transaction by" + "Error while starting database <{db}> transaction by" " driver <{driver}>." ).format(db=self._database, driver=self._driver_name) ) def _commit_transaction(self): """Commit DB transaction.""" - ret = self._c_funcs["db_commit_transaction"](self._pdriver) - if ret != self._c_funcs["DB_OK"]: - self._c_funcs["db_close_database_shutdown_driver"](self._pdriver) + ret = self._c_interface.db_commit_transaction(self._pdriver) + if ret != self._c_interface.DB_OK: + self._shutdown_driver() fatal( _( "Error while commit database <{db}> transaction" " by driver <{driver}>." ).format(db=self._database, driver=self._driver_name) ) - self._c_funcs["db_close_database_shutdown_driver"](self._pdriver) def _execute(self, sql): """Execute SQL @@ -354,17 +353,21 @@ def _execute(self, sql): :param str|list|tuple sql: SQL command string or list of SQLs commands """ - stmt = self._c_funcs["dbString"]() - self._c_funcs["db_init_string"](byref(stmt)) - self._c_funcs["db_set_string"](byref(stmt), sql) + stmt = self._c_interface.dbString() + self._c_interface.db_init_string(byref(stmt)) + self._c_interface.db_set_string(byref(stmt), sql) if ( - self._c_funcs["db_execute_immediate"](self._pdriver, byref(stmt)) - != self._c_funcs["DB_OK"] + self._c_interface.db_execute_immediate(self._pdriver, byref(stmt)) + != self._c_interface.DB_OK ): - self._c_funcs["db_free_string"](byref(stmt)) - self._c_funcs["db_close_database_shutdown_driver"](self._pdriver) + self._c_interface.db_free_string(byref(stmt)) + self._shutdown_driver() fatal(_("Error while executing SQL <{}>.").format(sql)) - self._c_funcs["db_free_string"](byref(stmt)) + self._c_interface.db_free_string(byref(stmt)) + + def _shutdown_driver(self): + """Close DB and shutdown driver""" + self._c_interface.db_close_database_shutdown_driver(self._pdriver) def execute(self, sql): """Execute SQL @@ -372,6 +375,9 @@ def execute(self, sql): :param str|list|tuple sql: SQL command string or list of SQLs statement """ + self._c_interface.G_gisinit("") + self._init_driver() + # Begin DB transaction self._begin_transaction() # Execute SQL string @@ -382,3 +388,5 @@ def execute(self, sql): self._execute(sql) # Commit DB transaction self._commit_transaction() + + self._shutdown_driver() diff --git a/scripts/db.dropcolumn/db.dropcolumn.py b/scripts/db.dropcolumn/db.dropcolumn.py index b55b863322d..2dcdef5682d 100755 --- a/scripts/db.dropcolumn/db.dropcolumn.py +++ b/scripts/db.dropcolumn/db.dropcolumn.py @@ -45,13 +45,12 @@ import sys import string -from grass.exceptions import CalledModuleError import grass.script as gscript +from grass.exceptions import CalledModuleError +from grass.script.db import DBHandler def main(): - from grass.script.db import DBHandler - table = options["table"] column = options["column"] database = options["database"] diff --git a/scripts/v.db.addcolumn/v.db.addcolumn.py b/scripts/v.db.addcolumn/v.db.addcolumn.py index 774bbfebfdd..cd6f46c66e1 100755 --- a/scripts/v.db.addcolumn/v.db.addcolumn.py +++ b/scripts/v.db.addcolumn/v.db.addcolumn.py @@ -42,14 +42,12 @@ import re -from grass.exceptions import CalledModuleError import grass.script as grass +from grass.exceptions import CalledModuleError +from grass.script.db import DBHandler def main(): - from grass.script.db import DBHandler - - global rm_files map = options["map"] layer = options["layer"] columns = options["columns"] diff --git a/scripts/v.db.dropcolumn/v.db.dropcolumn.py b/scripts/v.db.dropcolumn/v.db.dropcolumn.py index 7172ae541da..1c8d99dd6aa 100755 --- a/scripts/v.db.dropcolumn/v.db.dropcolumn.py +++ b/scripts/v.db.dropcolumn/v.db.dropcolumn.py @@ -39,11 +39,10 @@ import grass.script as grass from grass.exceptions import CalledModuleError +from grass.script.db import DBHandler def main(): - from grass.script.db import DBHandler - map = options["map"] layer = options["layer"] columns = options["columns"].split(",") diff --git a/scripts/v.db.join/v.db.join.py b/scripts/v.db.join/v.db.join.py index 4ebf6551b6b..2633491c4e5 100755 --- a/scripts/v.db.join/v.db.join.py +++ b/scripts/v.db.join/v.db.join.py @@ -67,11 +67,10 @@ import grass.script as gs from grass.exceptions import CalledModuleError +from grass.script.db import DBHandler def main(): - from grass.script.db import DBHandler - # Include mapset into the name, so we avoid multiple messages about # found in more mapsets. The following generates an error message, while the code # above does not. However, the above checks that the map exists, so we don't diff --git a/scripts/v.rast.stats/v.rast.stats.py b/scripts/v.rast.stats/v.rast.stats.py index d26e7aec919..be28c47d14a 100644 --- a/scripts/v.rast.stats/v.rast.stats.py +++ b/scripts/v.rast.stats/v.rast.stats.py @@ -76,9 +76,11 @@ import sys import os import atexit + import grass.script as grass -from grass.script.utils import decode from grass.exceptions import CalledModuleError +from grass.script.db import DBHandler +from grass.script.utils import decode def cleanup(): @@ -93,8 +95,6 @@ def cleanup(): def main(): - from grass.script.db import DBHandler - global tmp, sqltmp, tmpname, nuldev, vector, rastertmp rastertmp = False # setup temporary files From d42bc60f2c7957ec90c1d121a01e56c0524da031 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Mon, 11 Nov 2024 00:47:07 +0000 Subject: [PATCH 18/19] style: Remove unused pathlib import --- scripts/v.db.addcolumn/v.db.addcolumn.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/v.db.addcolumn/v.db.addcolumn.py b/scripts/v.db.addcolumn/v.db.addcolumn.py index b441a9a7e5e..31ad44bc9de 100755 --- a/scripts/v.db.addcolumn/v.db.addcolumn.py +++ b/scripts/v.db.addcolumn/v.db.addcolumn.py @@ -41,7 +41,6 @@ # %end import re -from pathlib import Path import grass.script as gs from grass.exceptions import CalledModuleError From dd5ca0a9f49c3d5e94189830a6be0e46e92d3317 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Mon, 11 Nov 2024 00:54:30 +0000 Subject: [PATCH 19/19] typing: Change type in docstring to typing annotations --- python/grass/script/db.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/python/grass/script/db.py b/python/grass/script/db.py index d1837872e2c..3b6adcb1c3f 100644 --- a/python/grass/script/db.py +++ b/python/grass/script/db.py @@ -19,6 +19,8 @@ .. sectionauthor:: Martin Landa """ +from __future__ import annotations + import os from ctypes import byref @@ -256,11 +258,11 @@ class DBHandler: ::execute """ - def __init__(self, driver_name, database): + def __init__(self, driver_name: str, database: str) -> None: """Constructor - :param str driver_name: DB driver name - :param str database: database name + :param driver_name: DB driver name + :param database: database name """ self._driver_name = driver_name self._database = database @@ -347,11 +349,10 @@ def _commit_transaction(self): ).format(db=self._database, driver=self._driver_name) ) - def _execute(self, sql): + def _execute(self, sql: str | list | tuple) -> None: """Execute SQL - :param str|list|tuple sql: SQL command string or list of SQLs - commands + :param sql: SQL command string or list of SQLs commands """ stmt = self._c_interface.dbString() self._c_interface.db_init_string(byref(stmt)) @@ -369,11 +370,10 @@ def _shutdown_driver(self): """Close DB and shutdown driver""" self._c_interface.db_close_database_shutdown_driver(self._pdriver) - def execute(self, sql): + def execute(self, sql: str | list | tuple) -> None: """Execute SQL - :param str|list|tuple sql: SQL command string or list of SQLs - statement + :param sql: SQL command string or list of SQLs statement """ self._c_interface.G_gisinit("") self._init_driver()