diff --git a/.cspell.json b/.cspell.json index 035b587f..ba51fa74 100644 --- a/.cspell.json +++ b/.cspell.json @@ -160,6 +160,9 @@ "hrsh", "pasky", "nvimrc", + "geoe", + "eplex", + "Queryability", "PYTHONPATH" ] } diff --git a/.gitignore b/.gitignore index 07b5bd3c..2649d119 100644 --- a/.gitignore +++ b/.gitignore @@ -114,3 +114,4 @@ geoe3/extlibs/* geest/extlibs/* !geest/extlibs/.gitkeep PROMPT.log +log.txt diff --git a/.nvim.lua2 b/.nvim.lua2 new file mode 100644 index 00000000..bf0745b8 --- /dev/null +++ b/.nvim.lua2 @@ -0,0 +1,298 @@ +-- GEEST nvim project configuration +-- Auto-sourced by nvim when exrc is enabled, or source manually with: +-- :source .nvim.lua + +-- Guard against re-sourcing +if vim.g.geest_loaded then + return +end +vim.g.geest_loaded = true + +-- Helper to run commands in a floating terminal +local function float_term(cmd, opts) + opts = opts or {} + local buf = vim.api.nvim_create_buf(false, true) + local width = opts.width or math.floor(vim.o.columns * 0.8) + local height = opts.height or math.floor(vim.o.lines * 0.8) + local row = math.floor((vim.o.lines - height) / 2) + local col = math.floor((vim.o.columns - width) / 2) + + local win = vim.api.nvim_open_win(buf, true, { + relative = 'editor', + width = width, + height = height, + row = row, + col = col, + style = 'minimal', + border = 'rounded', + title = opts.title or ' Terminal ', + title_pos = 'center', + }) + + if cmd then + vim.fn.termopen(cmd, { + on_exit = function(_, exit_code) + if opts.close_on_success and exit_code == 0 then + vim.defer_fn(function() + if vim.api.nvim_win_is_valid(win) then + vim.api.nvim_win_close(win, true) + end + end, 1000) + end + end, + }) + else + vim.fn.termopen(vim.o.shell) + end + vim.cmd('startinsert') +end + +-- Helper to open a bottom split terminal tailing the GEEST log file +local function open_log_tail() + local tmp_dir = os.getenv("TMPDIR") or os.getenv("TMP") or os.getenv("TEMP") or "/tmp" + local datestamp = os.date("%Y%m%d") + local log_file = tmp_dir .. "/geest_logfile_" .. datestamp .. ".log" + + -- Check if GEEST_LOG env var is set + local geest_log_env = os.getenv("GEEST_LOG") + if geest_log_env and geest_log_env ~= "" and geest_log_env ~= "0" then + log_file = geest_log_env + end + + -- Create a horizontal split at the bottom + vim.cmd('botright 12split') + local buf = vim.api.nvim_create_buf(false, true) + vim.api.nvim_win_set_buf(0, buf) + vim.api.nvim_buf_set_name(buf, 'GEEST Log') + + -- Set buffer options + vim.bo[buf].buftype = 'nofile' + vim.bo[buf].bufhidden = 'wipe' + vim.bo[buf].swapfile = false + + -- Start tailing the log file (create if it doesn't exist) + vim.fn.termopen(string.format('touch "%s" && tail -f "%s"', log_file, log_file), { + on_exit = function() + -- Buffer will be wiped automatically due to bufhidden setting + end, + }) + + -- Set window title + vim.wo.winfixheight = true + vim.wo.statusline = '%#StatusLine# GEEST Log: ' .. log_file .. ' %=' + + -- Return to the previous window + vim.cmd('wincmd p') +end + +-- Helper to launch QGIS with log tail (QGIS runs in background) +local function launch_qgis_with_log(cmd, title) + -- Open the log tail panel first + open_log_tail() + -- Launch QGIS as a background job (not in a terminal) + vim.fn.jobstart(cmd, { + detach = true, + on_exit = function(_, exit_code) + if exit_code ~= 0 then + vim.notify(title .. ' exited with code ' .. exit_code, vim.log.levels.WARN) + end + end, + }) + vim.notify('Started' .. title .. '(background)', vim.log.levels.INFO) +end + +-- Project-specific commands +vim.api.nvim_create_user_command('GeestQgis', function() + launch_qgis_with_log('GEEST_DEBUG=0 GEEST_EXPERIMENTAL=0 RUNNING_ON_LOCAL=1 nix run .#default -- --profile GEEST2', ' QGIS ') +end, { desc = 'Launch QGIS (normal mode)' }) + +vim.api.nvim_create_user_command('GeestQgisDebug', function() + launch_qgis_with_log('GEEST_DEBUG=1 GEEST_EXPERIMENTAL=0 RUNNING_ON_LOCAL=1 nix run .#default -- --profile GEEST2', ' QGIS Debug ') +end, { desc = 'Launch QGIS (debug mode)' }) + +vim.api.nvim_create_user_command('GeestQgisExperimental', function() + launch_qgis_with_log('GEEST_DEBUG=0 GEEST_EXPERIMENTAL=1 RUNNING_ON_LOCAL=1 nix run .#default -- --profile GEEST2', ' QGIS Experimental ') +end, { desc = 'Launch QGIS (experimental features)' }) + +vim.api.nvim_create_user_command('GeestQgisLtr', function() + launch_qgis_with_log('RUNNING_ON_LOCAL=1 nix run .#qgis-ltr', ' QGIS LTR ') +end, { desc = 'Launch QGIS LTR' }) + +vim.api.nvim_create_user_command('GeestPrecommit', function() + float_term('pre-commit run --all-files', { title = ' Pre-commit ' }) +end, { desc = 'Run pre-commit checks' }) + +vim.api.nvim_create_user_command('GeestPrecommitStaged', function() + float_term('pre-commit run', { title = ' Pre-commit (staged) ' }) +end, { desc = 'Run pre-commit on staged files' }) + +vim.api.nvim_create_user_command('GeestTests', function() + float_term('./scripts/run-tests.sh', { title = ' Tests ' }) +end, { desc = 'Run tests' }) + +vim.api.nvim_create_user_command('GeestClean', function() + float_term('./scripts/clean.sh', { title = ' Clean ' }) +end, { desc = 'Clean build artifacts' }) + +vim.api.nvim_create_user_command('GeestRemovePycache', function() + float_term('./scripts/remove_pycache.sh', { title = ' Remove __pycache__ ' }) +end, { desc = 'Remove __pycache__ directories' }) + +vim.api.nvim_create_user_command('GeestDocstrings', function() + float_term('./scripts/docstrings_check.sh', { title = ' Docstrings Check ' }) +end, { desc = 'Check docstrings' }) + +vim.api.nvim_create_user_command('GeestEncoding', function() + float_term('./scripts/encoding_check.sh', { title = ' Encoding Check ' }) +end, { desc = 'Check file encodings' }) + +vim.api.nvim_create_user_command('GeestGource', function() + float_term('./scripts/gource.sh', { title = ' Gource ' }) +end, { desc = 'Run gource visualization' }) + +vim.api.nvim_create_user_command('GeestCompileStrings', function() + float_term('./scripts/compile-strings.sh', { title = ' Compile Strings ' }) +end, { desc = 'Compile translation strings' }) + +vim.api.nvim_create_user_command('GeestUpdateStrings', function() + float_term('./scripts/update-strings.sh', { title = ' Update Strings ' }) +end, { desc = 'Update translation strings' }) + +vim.api.nvim_create_user_command('GeestTerm', function() + float_term(nil, { title = ' Terminal ' }) +end, { desc = 'Open floating terminal' }) + +vim.api.nvim_create_user_command('GeestLogTail', function() + open_log_tail() +end, { desc = 'Open GEEST log tail panel' }) + +vim.api.nvim_create_user_command('GeestGitStatus', function() + float_term('git status && echo "\\n--- Recent commits ---\\n" && git log --oneline -10', { title = ' Git Status ' }) +end, { desc = 'Git status and recent commits' }) + +vim.api.nvim_create_user_command('GeestGitDiff', function() + float_term('git diff', { title = ' Git Diff ' }) +end, { desc = 'Git diff' }) + +vim.api.nvim_create_user_command('GeestGitLog', function() + float_term('git log --oneline --graph --decorate -30', { title = ' Git Log ' }) +end, { desc = 'Git log (graph)' }) + +vim.api.nvim_create_user_command('GeestLazygit', function() + float_term('lazygit', { title = ' Lazygit ' }) +end, { desc = 'Open lazygit' }) + +-- Build & Release commands +vim.api.nvim_create_user_command('GeestBuild', function() + float_term('python admin.py build', { title = ' Build Plugin ' }) +end, { desc = 'Build plugin to build/' }) + +vim.api.nvim_create_user_command('GeestGenerateZip', function() + float_term('python admin.py generate-zip', { title = ' Generate ZIP ' }) +end, { desc = 'Generate plugin ZIP' }) + +vim.api.nvim_create_user_command('GeestInstall', function() + float_term('python admin.py --qgis-profile GEEST2 install', { title = ' Install Plugin ' }) +end, { desc = 'Install plugin to QGIS profile' }) + +vim.api.nvim_create_user_command('GeestUninstall', function() + float_term('python admin.py --qgis-profile GEEST2 uninstall', { title = ' Uninstall Plugin ' }) +end, { desc = 'Uninstall plugin from QGIS profile' }) + +vim.api.nvim_create_user_command('GeestSymlink', function() + float_term('python admin.py --qgis-profile GEEST2 symlink', { title = ' Symlink Plugin ' }) +end, { desc = 'Symlink plugin to QGIS profile' }) + +vim.api.nvim_create_user_command('GeestGenerateRepoXml', function() + float_term('python admin.py generate-plugin-repo-xml', { title = ' Generate Repo XML ' }) +end, { desc = 'Generate plugin repository XML' }) + +vim.api.nvim_create_user_command('GeestBundleDeps', function() + float_term('python admin.py bundle-deps', { title = ' Bundle Dependencies ' }) +end, { desc = 'Bundle vendored dependencies (h3, etc.)' }) + +vim.api.nvim_create_user_command('GeestCleanExtlibs', function() + float_term('python admin.py clean-extlibs', { title = ' Clean Extlibs ' }) +end, { desc = 'Clean vendored dependencies' }) + +vim.api.nvim_create_user_command('GeestReleaseDraft', function() + float_term('gh release create --draft --generate-notes', { title = ' Draft Release ' }) +end, { desc = 'Create draft GitHub release' }) + +vim.api.nvim_create_user_command('GeestReleaseList', function() + float_term('gh release list', { title = ' Releases ' }) +end, { desc = 'List GitHub releases' }) + +vim.api.nvim_create_user_command('GeestTagList', function() + float_term('git tag -l --sort=-v:refname | head -20', { title = ' Tags ' }) +end, { desc = 'List recent tags' }) + +vim.api.nvim_create_user_command('GeestTagCreate', function() + vim.ui.input({ prompt = 'Tag version (e.g., v2.0.1): ' }, function(tag) + if tag and tag ~= '' then + vim.ui.input({ prompt = 'Tag message: ' }, function(msg) + if msg and msg ~= '' then + float_term(string.format('git tag -a %s -m "%s" && echo "Tag %s created. Push with: git push origin %s"', tag, msg, tag, tag), { title = ' Create Tag ' }) + end + end) + end + end) +end, { desc = 'Create annotated tag' }) + +-- Register with which-key under p (Project) +local wk_ok, wk = pcall(require, 'which-key') +if wk_ok then + wk.add({ + { 'p', group = 'Project' }, + -- QGIS launchers + { 'pq', group = 'QGIS' }, + { 'pqq', 'GeestQgis', desc = 'Launch QGIS' }, + { 'pqd', 'GeestQgisDebug', desc = 'Launch QGIS (debug)' }, + { 'pqe', 'GeestQgisExperimental', desc = 'Launch QGIS (experimental)' }, + { 'pql', 'GeestQgisLtr', desc = 'Launch QGIS LTR' }, + -- Pre-commit / Quality + { 'pc', group = 'Checks' }, + { 'pcc', 'GeestPrecommit', desc = 'Pre-commit (all files)' }, + { 'pcs', 'GeestPrecommitStaged', desc = 'Pre-commit (staged)' }, + { 'pcd', 'GeestDocstrings', desc = 'Check docstrings' }, + { 'pce', 'GeestEncoding', desc = 'Check encodings' }, + -- Tests + { 'pt', 'GeestTests', desc = 'Run tests' }, + -- Clean + { 'px', group = 'Clean' }, + { 'pxc', 'GeestClean', desc = 'Clean build artifacts' }, + { 'pxp', 'GeestRemovePycache', desc = 'Remove __pycache__' }, + -- Translations + { 'pi', group = 'i18n' }, + { 'pic', 'GeestCompileStrings', desc = 'Compile strings' }, + { 'piu', 'GeestUpdateStrings', desc = 'Update strings' }, + -- Git + { 'pg', group = 'Git' }, + { 'pgs', 'GeestGitStatus', desc = 'Status + recent commits' }, + { 'pgd', 'GeestGitDiff', desc = 'Diff' }, + { 'pgl', 'GeestGitLog', desc = 'Log (graph)' }, + { 'pgg', 'GeestLazygit', desc = 'Lazygit' }, + -- Build & Package + { 'pb', group = 'Build' }, + { 'pbb', 'GeestBuild', desc = 'Build plugin' }, + { 'pbz', 'GeestGenerateZip', desc = 'Generate ZIP' }, + { 'pbi', 'GeestInstall', desc = 'Install to QGIS' }, + { 'pbu', 'GeestUninstall', desc = 'Uninstall from QGIS' }, + { 'pbs', 'GeestSymlink', desc = 'Symlink to QGIS' }, + { 'pbx', 'GeestGenerateRepoXml', desc = 'Generate repo XML' }, + { 'pbd', 'GeestBundleDeps', desc = 'Bundle dependencies' }, + { 'pbc', 'GeestCleanExtlibs', desc = 'Clean extlibs' }, + -- Release + { 'pr', group = 'Release' }, + { 'prd', 'GeestReleaseDraft', desc = 'Draft GitHub release' }, + { 'prl', 'GeestReleaseList', desc = 'List releases' }, + { 'prt', 'GeestTagList', desc = 'List tags' }, + { 'prn', 'GeestTagCreate', desc = 'Create new tag' }, + -- Misc + { 'pv', 'GeestGource', desc = 'Gource visualization' }, + { 'pp', 'GeestTerm', desc = 'Floating terminal' }, + { 'pl', 'GeestLogTail', desc = 'Tail GEEST log' }, + }) +end + +vim.notify("GEEST: Project commands and p menu loaded", vim.log.levels.INFO) diff --git a/docs/userguide/datacollection.md b/docs/userguide/datacollection.md index 35eb2b38..6b8f6d3c 100644 --- a/docs/userguide/datacollection.md +++ b/docs/userguide/datacollection.md @@ -271,6 +271,12 @@ While the above table showcases data specific to Saint Lucia, similar data can b 8. Mapillary: A collaborative platform that offers street-level imagery contributed by users worldwide. Mapillary data includes vector data on street crossings, sidewalks, and public lighting, making it useful for place-based and accessibility assessments.

+## Data Attribution and Citation + +GeoE3 may use external datasets and services depending on the selected workflows and user-provided inputs. Typical sources include OpenStreetMap, OpenRouteService, GHSL (European Commission JRC), Ookla Open Data, Space2Stats, ACLED, and VIIRS Nighttime Lights. + +When sharing outputs, always verify and include the relevant provider attribution, citation text, and license/terms for the datasets actually used in your analysis. + ### Instructions for Data Collection: - **Query the Source**: Use the query instructions provided in the table to filter and collect specific data. diff --git a/docs/workflow_analysis.md b/docs/workflow_analysis.md new file mode 100644 index 00000000..44b5001b --- /dev/null +++ b/docs/workflow_analysis.md @@ -0,0 +1,286 @@ +# GeoE3 Workflow Analysis: Raster vs Vector Processing + +This document analyzes all workflows in the GeoE3 plugin to identify which can be migrated from raster-based processing to pure vector/SQL operations on the study_area_grid layer. + +## Workflow Analysis Table + +| Type | ID | Workflow Option | Raster Only (Current) | Vector-Only Possible? | Notes | +| ------------------------------------ | ------------------------------------------------------- | ---------------------------------------- | --------------------- | --------------------- | ----------------------------------- | --- | +| **CONTEXTUAL DIMENSION** | | | | | | | +| Dimension | contextual | Dimension aggregation | Yes | **Yes** | `SUM(factor * weight)` SQL | +| Factor | eplex | Factor aggregation | Yes | **Yes** | `SUM(indicator * weight)` SQL | +| Factor | workplace_discrimination | Factor aggregation | Yes | **Yes** | SQL weighted average | +| Factor | regulatory_frameworks | Factor aggregation | Yes | **Yes** | SQL weighted average | +| Factor | financial_inclusion | Factor aggregation | Yes | **Yes** | SQL weighted average | +| Indicator | eplex_score_indicator | use_eplex_score | No | **Yes** | Uniform scalar → set attribute | +| Indicator | eplex_score_indicator | use_index_score | No | **Yes** | Uniform scalar | +| Indicator | eplex_score_indicator | use_contextual_index_score | No | **Yes** | Rescaled scalar | +| Indicator | Workplace_Index | use_contextual_index_score | No | **Yes** | Rescaled scalar | +| Indicator | Pay_Parenthood_Index | use_contextual_index_score | No | **Yes** | Rescaled scalar | +| Indicator | Entrepreneurship_Index | use_contextual_index_score | No | **Yes** | Rescaled scalar | +| **ACCESSIBILITY DIMENSION** | | | | | | | +| Dimension | accessibility | Dimension aggregation | Yes | **Yes** | `SUM(factor * weight)` SQL | +| Factor | women_s_travel_patterns | Factor aggregation | Yes | **Yes** | SQL weighted avg of 5 indicators | +| Factor | access_to_public_transport | Factor aggregation | Yes | **Yes** | SQL weighted average | +| Factor | access_to_health_facilities | Factor aggregation | Yes | **Yes** | SQL weighted average | +| Factor | access_to_education_and_training_facilities | Factor aggregation | Yes | **Yes** | SQL weighted average | +| Factor | access_to_financial_facilities | Factor aggregation | Yes | **Yes** | SQL weighted average | +| Indicator | Kindergartens_Location | use_multi_buffer_point | No | **Yes** | Spatial join: grid ∩ buffers | +| Indicator | Kindergartens_Location | use_single_buffer_point | No | **Yes** | Spatial join | +| Indicator | Kindergartens_Location | use_point_per_cell | No | **Yes** | Count points in cell | +| Indicator | Primary_School_Location | use_multi_buffer_point | No | **Yes** | Spatial join | +| Indicator | Primary_School_Location | use_point_per_cell | No | **Yes** | Count points in cell | +| Indicator | Groceries_Location | use_multi_buffer_point | No | **Yes** | Spatial join | +| Indicator | Groceries_Location | use_point_per_cell | No | **Yes** | Count points in cell | +| Indicator | Pharmacies_Location | use_multi_buffer_point | No | **Yes** | Spatial join | +| Indicator | Pharmacies_Location | use_point_per_cell | No | **Yes** | Count points in cell | +| Indicator | Green_Space_location | use_multi_buffer_point | No | **Yes** | Spatial join | +| Indicator | Green_Space_location | use_polygon_per_cell | No | **Yes** | Polygon intersection | +| Indicator | Public_Transport_location | use_multi_buffer_point | No | **Yes** | Spatial join | +| Indicator | Public_Transport_location | use_point_per_cell | No | **Yes** | Count points in cell | +| Indicator | Hospital_Location | use_multi_buffer_point | No | **Yes** | Spatial join | +| Indicator | Hospital_Location | use_point_per_cell | No | **Yes** | Count points in cell | +| Indicator | Universities_Location | use_multi_buffer_point | No | **Yes** | Spatial join | +| Indicator | Universities_Location | use_point_per_cell | No | **Yes** | Count points in cell | +| Indicator | Banks_Location | use_multi_buffer_point | No | **Yes** | Spatial join | +| Indicator | Banks_Location | use_point_per_cell | No | **Yes** | Count points in cell | +| **PLACE CHARACTERIZATION DIMENSION** | | | | | | | +| Dimension | place_characterization | Dimension aggregation | Yes | **Yes** | `SUM(factor * weight)` SQL | +| Factor | active_transport | Factor aggregation | Yes | **Yes** | SQL weighted average | +| Factor | safety_perception | Factor aggregation | Yes | **Yes** | SQL weighted average | +| Factor | fcv | Factor aggregation | Yes | **Yes** | SQL weighted average | +| Factor | education | Factor aggregation | Yes | **Yes** | SQL weighted average | +| Factor | digital_inclusion | Factor aggregation | Yes | **Yes** | SQL weighted average | +| Factor | environmental_hazards | Factor aggregation | Yes | **Yes** | SQL weighted avg of 5 hazards | +| Factor | water_sanitation | Factor aggregation | Yes | **Yes** | SQL weighted average | +| Indicator | Active_Transport_Network | use_polyline_per_cell | No | **Yes** | Polyline length per cell | +| Indicator | Active_Transport_Network | use_osm_transport_polyline_per_cell | No | **Yes** | OSM highway scoring per cell | +| Indicator | Street_Lights | use_nighttime_lights | **Yes** | No | GHSL nighttime lights raster | +| Indicator | Street_Lights | use_street_lights | Maybe | Maybe | Point buffers=vector, raster=raster | +| Indicator | Street_Lights | use_classify_safety_polygon_into_classes | No | **Yes** | Polygon classification | +| Indicator | FCV | use_csv_to_point_layer | No | **Yes** | CSV→points→buffer→intersect | +| Indicator | FCV | use_single_buffer_point | No | **Yes** | Spatial join with buffer | +| Indicator | FCV | use_point_per_cell | No | **Yes** | Count points in cell | +| Indicator | Education | use_index_score_with_ghsl | **Yes** | No | Requires GHSL raster mask | +| Indicator | Education | use_classify_polygon_into_classes | No | **Yes** | Polygon classification | +| Indicator | Digital_Inclusion | use_index_score_with_ookla | **Yes** | No | Requires Ookla raster | +| Indicator | Digital_Inclusion | use_classify_polygon_into_classes | No | **Yes** | Polygon classification | +| Indicator | Fire | use_environmental_hazards | **Yes** | No | Hazard raster input | +| Indicator | Flood | use_environmental_hazards | **Yes** | No | Hazard raster input | +| Indicator | Landslide | use_environmental_hazards | **Yes** | No | Hazard raster input | +| Indicator | Cyclone | use_environmental_hazards | **Yes** | No | Hazard raster input | +| Indicator | Drought | use_environmental_hazards | **Yes** | No | Hazard raster input | +| Indicator | Water_Sanitation | use_single_buffer_point | No | **Yes** | Spatial join with 3km buffer | +| Indicator | Water_Sanitation | use_point_per_cell | No | **Yes** | Count points in cell | +| **ANALYSIS RESULTS** | | | | | | | +| Analysis | geoe3_score | Analysis aggregation | Yes | **Yes** | `SUM(dimension * weight)` SQL | +| Analysis | geoe3_by_population | Population weighting | Yes | **Yes** | `geoe3_score * population` SQL | +| Analysis | geoe3_score_ghsl_masked | GHSL masking | **Yes** | No | Requires GHSL settlement raster | +| Analysis | geoe3_by_population_ghsl_masked | GHSL + population | **Yes** | No | Requires GHSL settlement raster | +| Analysis | geoe3_score_subnational_aggregation | Subnational stats | Yes | **Yes** | SQL GROUP BY subnational unit | +| Analysis | geoe3_by_population_subnational_aggregation | Subnational + pop | Yes | **Yes** | SQL GROUP BY with population | +| Analysis | geoe3_score_ghsl_masked_subnational_aggregation | GHSL + subnational | **Yes** | No | Requires GHSL raster first | +| Analysis | geoe3_by_population_ghsl_masked_subnational_aggregation | All combined | **Yes** | No | Requires GHSL raster first | +| Analysis | geoe3_by_population_by_opportunities_mask | Opportunities mask | **Yes** | No | Requires opportunities raster | +| Analysis | population | Population per cell | **Yes** | No | Sample population raster | + +--- + +## Summary by Type + +| Type | Total | Raster Required | Vector-Only Possible | +| --------- | --------------------- | ----------------- | ------------------------- | +| Dimension | 3 | 3 (current impl) | **3** (all migratable) | +| Factor | 16 | 16 (current impl) | **16** (all migratable) | +| Indicator | 21 IDs, ~40 workflows | 8 workflows | **32+ workflows** | +| Analysis | 10 | 5 | **5** (partial migration) | +| **TOTAL** | ~70 workflows | ~32 | **~56 migratable** | + +--- + +## Raster-Only Dependencies + +The following workflows genuinely require raster data and cannot be converted to pure vector operations: + +| Workflow | Raster Data Required | Reason | +| ------------------------------ | ---------------------------------------- | --------------------------------- | +| use_nighttime_lights | GHSL Nighttime Lights | Input is raster imagery | +| use_environmental_hazards (5x) | Fire, Flood, Landslide, Cyclone, Drought | Input hazard data is raster | +| use_index_score_with_ghsl | GHSL Settlement Layer | Mask requires raster intersection | +| use_index_score_with_ookla | Ookla Internet Coverage | Coverage data is raster | +| population | Population Raster | WorldPop/GHSL population grids | +| geoe3\_\*\_ghsl_masked | GHSL Settlement Layer | Masking requires raster | +| geoe3\_\*\_opportunities_mask | Opportunities Raster | Masking requires raster | + +--- + +## SQL Examples for Vector-Only Workflows + +### Indicator: Index Score (Uniform Value) + +```sql +-- Set a uniform score across all grid cells +UPDATE study_area_grid +SET eplex_score_indicator = 3.5 +WHERE area_name = 'Study Area 1'; +``` + +### Indicator: Multi-Buffer Point (Spatial Join) + +```sql +-- Score grid cells based on proximity to points (e.g., kindergartens) +-- Assumes buffers have been pre-computed with scores +UPDATE study_area_grid g +SET kindergartens_location = COALESCE( + (SELECT MAX(b.score) + FROM kindergarten_buffers b + WHERE ST_Intersects(g.geom, b.geom)), + 0 +) +WHERE area_name = 'Study Area 1'; +``` + +### Indicator: Point Per Cell (Count) + +```sql +-- Count features within each grid cell +UPDATE study_area_grid g +SET water_sanitation = ( + SELECT COUNT(*) + FROM water_points p + WHERE ST_Contains(g.geom, p.geom) +) +WHERE area_name = 'Study Area 1'; +``` + +### Indicator: Polyline Per Cell (Length/Score) + +```sql +-- Calculate walkability score based on road types in cell +UPDATE study_area_grid g +SET active_transport_network = COALESCE( + (SELECT MAX( + CASE + WHEN r.highway IN ('footway', 'pedestrian', 'cycleway') THEN 5 + WHEN r.highway IN ('residential', 'living_street') THEN 4 + WHEN r.highway IN ('tertiary', 'unclassified') THEN 3 + WHEN r.highway IN ('secondary') THEN 2 + WHEN r.highway IN ('primary', 'trunk') THEN 1 + ELSE 0 + END + ) + FROM osm_roads r + WHERE ST_Intersects(g.geom, r.geom)), + 0 +) +WHERE area_name = 'Study Area 1'; +``` + +### Factor Aggregation (Weighted Average) + +```sql +-- Aggregate indicators into factor score +UPDATE study_area_grid +SET women_s_travel_patterns = ( + kindergartens_location * 0.2 + + primary_school_location * 0.2 + + groceries_location * 0.2 + + pharmacies_location * 0.2 + + green_space_location * 0.2 +) +WHERE area_name = 'Study Area 1'; +``` + +### Dimension Aggregation (Weighted Average) + +```sql +-- Aggregate factors into dimension score +UPDATE study_area_grid +SET accessibility = ( + women_s_travel_patterns * 0.2 + + access_to_public_transport * 0.2 + + access_to_health_facilities * 0.2 + + access_to_education_and_training_facilities * 0.2 + + access_to_financial_facilities * 0.2 +) +WHERE area_name = 'Study Area 1'; +``` + +### Analysis: GeoE3 Score (Final Aggregation) + +```sql +-- Calculate final GeoE3 score from dimensions +UPDATE study_area_grid +SET geoe3_score = ( + contextual * 0.1 + + accessibility * 0.45 + + place_characterization * 0.45 +) +WHERE area_name = 'Study Area 1'; +``` + +### Analysis: Population Weighted Score + +```sql +-- Calculate population-weighted score +UPDATE study_area_grid +SET geoe3_by_population = geoe3_score * population +WHERE area_name = 'Study Area 1'; +``` + +### Analysis: Subnational Aggregation + +```sql +-- Aggregate scores by subnational unit +SELECT + subnational_unit, + AVG(geoe3_score) as avg_score, + SUM(geoe3_by_population) / SUM(population) as pop_weighted_avg, + MIN(geoe3_score) as min_score, + MAX(geoe3_score) as max_score, + COUNT(*) as cell_count +FROM study_area_grid +WHERE area_name = 'Study Area 1' +GROUP BY subnational_unit; +``` + +--- + +## Migration Benefits + +Converting from raster-based to vector-based processing provides: + +1. **Performance**: SQL operations on indexed vector tables are significantly faster than pixel-by-pixel raster sampling +2. **Simplicity**: No intermediate raster files to manage +3. **Accuracy**: Values stored directly in grid cells without resampling artifacts +4. **Storage**: Reduced disk usage (no duplicate raster outputs) +5. **Queryability**: Results immediately available for SQL analysis and reporting + +--- + +## Implementation Priority + +### Phase 1: Aggregation Workflows (Highest Impact) + +- Factor aggregation (16 workflows) +- Dimension aggregation (3 workflows) +- Analysis aggregation (geoe3_score, geoe3_by_population) + +### Phase 2: Vector-Based Indicators + +- Index score workflows (uniform values) +- Multi-buffer point workflows (spatial joins) +- Single-buffer point workflows +- Point/polyline/polygon per cell workflows + +### Phase 3: Hybrid Workflows + +- Workflows requiring both raster sampling AND vector output +- Environmental hazards (sample raster → write to grid) +- Population (sample raster → write to grid) + +--- + +_Document generated: 2026-03-30_ + +_Made with 💗 by [Kartoza](https://kartoza.com) | [Donate](https://github.com/sponsors/worldbank/GEOE3) | [GitHub](https://github.com/worldbank/GEOE3)_ diff --git a/geest/__init__.py b/geest/__init__.py index 8576aaf0..2ec96ba5 100644 --- a/geest/__init__.py +++ b/geest/__init__.py @@ -39,13 +39,14 @@ import os import pstats import subprocess # nosec B404 +import sys import tempfile import unittest from shutil import which from typing import Optional from qgis.core import Qgis, QgsProject -from qgis.PyQt.QtCore import QSettings, Qt, pyqtSignal +from qgis.PyQt.QtCore import QEvent, QObject, QSettings, Qt, pyqtSignal from qgis.PyQt.QtGui import QIcon from qgis.PyQt.QtWidgets import ( QAction, @@ -84,6 +85,9 @@ format="%(asctime)s [%(levelname)s] %(message)s", level=logging.DEBUG, ) +# Suppress PyQt uic module debug spam (push/pop widget, setting property, etc.) +logging.getLogger("PyQt5.uic").setLevel(logging.WARNING) +logging.getLogger("PyQt6.uic").setLevel(logging.WARNING) date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") log_message("»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»", force=True) log_message(f"GeoE3 started at {date}", force=True) @@ -95,6 +99,25 @@ log_message("QGIS Version: {}".format(Qgis.QGIS_VERSION), force=True) +class CanvasOverlayFilter(QObject): + """Event filter to clear overlay label when clicking on map canvas.""" + + def eventFilter(self, obj, event): + """Filter mouse press events on map canvas to clear overlay. + + Args: + obj: The object that received the event. + event: The event to process. + + Returns: + False to let the event propagate normally. + """ + if event.type() == QEvent.MouseButtonPress and event.button() == Qt.LeftButton: + QSettings().setValue("geoe3/overlay_label", "") + QSettings().setValue("geoe3/pie_data", "") + return False + + def classFactory(iface): # pylint: disable=missing-function-docstring """🔄 Classfactory. @@ -163,8 +186,6 @@ def get_test_directory(self): Raises: ValueError: If GEOE3_TEST_DIR (or GEEST_TEST_DIR) is not set or points to invalid directory """ - import sys - # Get test directory from environment variable (with fallback for backward compatibility) env_test_dir = os.getenv("GEOE3_TEST_DIR") or os.getenv("GEEST_TEST_DIR") @@ -286,6 +307,10 @@ def initGui(self): # pylint: disable=missing-function-docstring self.iface.registerOptionsWidgetFactory(self.options_factory) self.setup_map_canvas_items() + # Install event filter to clear overlay label on canvas left-click + self._canvas_overlay_filter = CanvasOverlayFilter() + self.iface.mapCanvas().viewport().installEventFilter(self._canvas_overlay_filter) + def run_tests(self): """Run unit tests in the python console.""" @@ -334,7 +359,9 @@ def run_tests(self): def setup_map_canvas_items(self): """⚙️ Setup map canvas items.""" - self.label_overlay = LayerDescriptionItem(self.iface.mapCanvas()) + # TEMPORARY: Disable layer description overlay creation. + # self.label_overlay = LayerDescriptionItem(self.iface.mapCanvas()) + self.label_overlay = None experimental_features = int(os.getenv("GEOE3_EXPERIMENTAL") or os.getenv("GEEST_EXPERIMENTAL", 0)) if experimental_features: self.pie_overlay = PieChartItem(self.iface.mapCanvas()) @@ -755,6 +782,11 @@ def unload(self): # pylint: disable=missing-function-docstring self.iface.unregisterOptionsWidgetFactory(self.options_factory) self.options_factory = None + # Remove canvas event filter + if hasattr(self, "_canvas_overlay_filter") and self._canvas_overlay_filter: + self.iface.mapCanvas().viewport().removeEventFilter(self._canvas_overlay_filter) + self._canvas_overlay_filter = None + # Remove dock widget if it exists if self.dock_widget: self.iface.removeDockWidget(self.dock_widget) @@ -838,7 +870,8 @@ def debug(self): title="GeoE3", message=f"Visual Studio Code debugger is now attached on port {self.DEBUG_PORT}", ) - self.debug_action.setEnabled(False) # prevent user starting it twice + if self.debug_action: + self.debug_action.setEnabled(False) # prevent user starting it twice self.debug_running = True def run(self): diff --git a/geest/core/__init__.py b/geest/core/__init__.py index 8cc92ece..90ea440b 100644 --- a/geest/core/__init__.py +++ b/geest/core/__init__.py @@ -23,6 +23,7 @@ from .default_settings import default_settings from .json_tree_item import JsonTreeItem from .settings import set_setting, setting +from .s2s_task_gate import S2STaskGate from .workflow_queue_manager import WorkflowQueueManager # from .json_validator import JSONValidator diff --git a/geest/core/algorithms/area_iterator.py b/geest/core/algorithms/area_iterator.py index b07cbb8c..1ace5d22 100644 --- a/geest/core/algorithms/area_iterator.py +++ b/geest/core/algorithms/area_iterator.py @@ -140,14 +140,14 @@ def area_count(self) -> int: """ return self.total_features - def __iter__(self) -> Iterator[Tuple[QgsGeometry, QgsGeometry, float]]: + def __iter__(self) -> Iterator[Tuple[QgsGeometry, QgsGeometry, QgsGeometry, float, str]]: """ Iterator that yields pairs of geometries from the polygon layer and the corresponding bbox layer, - along with a progress percentage. + along with a progress percentage and area name. Yields: - Iterator[Tuple[QgsGeometry, QgsGeometry, float]]: Yields a tuple of polygon and bbox geometries, - along with a progress value representing the percentage of the iteration completed. + Iterator[Tuple[QgsGeometry, QgsGeometry, QgsGeometry, float, str]]: Yields a tuple of + polygon geometry, clip geometry, bbox geometry, progress percentage, and area name. """ try: # Ensure all layers have the same CRS @@ -224,8 +224,8 @@ def __iter__(self) -> Iterator[Tuple[QgsGeometry, QgsGeometry, float]]: level=Qgis.Info, ) - # Yield a tuple with polygon geometry, clip geometry, bbox geometry, and progress percentage - yield polygon_feature.geometry(), clip_geom, bbox_feature.geometry(), progress_percent + # Yield a tuple with polygon geometry, clip geometry, bbox geometry, progress percentage, and area name + yield polygon_feature.geometry(), clip_geom, bbox_feature.geometry(), progress_percent, area_name else: log_message( diff --git a/geest/core/algorithms/features_per_cell_processor.py b/geest/core/algorithms/features_per_cell_processor.py index 94621558..311d501e 100644 --- a/geest/core/algorithms/features_per_cell_processor.py +++ b/geest/core/algorithms/features_per_cell_processor.py @@ -277,6 +277,10 @@ def assign_values_to_grid(grid_layer: QgsVectorLayer, feedback: QgsFeedback = No END """ ds.ExecuteSQL(sql) + try: + ds.ExecuteSQL("PRAGMA wal_checkpoint(TRUNCATE)") + except Exception: # nosec B110 – non-fatal; the close will still flush + pass ds = None # Close the datasource log_message( diff --git a/geest/core/algorithms/ghsl_downloader.py b/geest/core/algorithms/ghsl_downloader.py index efe2c94a..ad37d5a6 100644 --- a/geest/core/algorithms/ghsl_downloader.py +++ b/geest/core/algorithms/ghsl_downloader.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -"""📦 Ghsl Downloader module. +"""📦 GHSL Downloader module. This module contains functionality for ghsl downloader. """ diff --git a/geest/core/algorithms/opportunities_by_wee_score_population_processor.py b/geest/core/algorithms/opportunities_by_wee_score_population_processor.py index cafdaf2b..54534992 100644 --- a/geest/core/algorithms/opportunities_by_wee_score_population_processor.py +++ b/geest/core/algorithms/opportunities_by_wee_score_population_processor.py @@ -179,7 +179,7 @@ def calculate_score(self) -> None: algebra and saves the result for each area. """ area_iterator = AreaIterator(self.study_area_gpkg_path) - for index, (_, _, _, _) in enumerate(area_iterator): + for index, (_, _, _, _, _) in enumerate(area_iterator): if self.isCanceled(): return diff --git a/geest/core/algorithms/opportunities_by_wee_score_processor.py b/geest/core/algorithms/opportunities_by_wee_score_processor.py index de2fa82a..435a4ba3 100644 --- a/geest/core/algorithms/opportunities_by_wee_score_processor.py +++ b/geest/core/algorithms/opportunities_by_wee_score_processor.py @@ -20,6 +20,7 @@ from geest.core import JsonTreeItem from geest.core.algorithms import AreaIterator from geest.core.constants import GDAL_OUTPUT_DATA_TYPE +from geest.core.grid_column_utils import clear_grid_column from geest.utilities import log_message, resources_path @@ -154,9 +155,10 @@ def validate_rasters( def calculate_score(self) -> None: """ Calculates Mask x GeoE3 Score using raster algebra and saves the result for each area. + Also writes the masked values to the study_area_grid column 'geoe3_masked'. """ area_iterator = AreaIterator(self.study_area_gpkg_path) - for index, (_, _, _, _) in enumerate(area_iterator): + for index, (_, _, _, _, area_name) in enumerate(area_iterator): if self.isCanceled(): return @@ -194,6 +196,36 @@ def calculate_score(self) -> None: log_message(f"Masked GeoE3 Score raster saved to {output_path}") + # Write masked values to grid using SQL — copy geoe3 scores only + # where the opportunities mask (settlements) is present, leaving + # cells outside settlements as NULL. + self._write_masked_to_grid() + + def _write_masked_to_grid(self) -> None: + """Copy geoe3 values to geoe3_masked for cells covered by the settlements mask. + + Uses a single SQL UPDATE rather than raster sampling, which is both + faster and correctly leaves non-settlement cells as NULL instead of 0. + """ + from osgeo import ogr + + clear_grid_column(self.study_area_gpkg_path, "geoe3_masked") + ds = ogr.Open(self.study_area_gpkg_path, 1) + if not ds: + log_message("Could not open GeoPackage to write geoe3_masked") + return + sql = ( + 'UPDATE study_area_grid SET "geoe3_masked" = "geoe3" ' # nosec B608 + 'WHERE "opportunities_mask" IS NOT NULL' + ) + ds.ExecuteSQL(sql) + try: + ds.ExecuteSQL("PRAGMA wal_checkpoint(TRUNCATE)") + except Exception: # nosec B110 + pass + ds = None + log_message("Updated geoe3_masked grid column from geoe3 where opportunities_mask is set") + def generate_vrt(self) -> str: """ Combines all GeoE3 Score rasters into a single VRT and applies a QML style. diff --git a/geest/core/algorithms/opportunities_mask_processor.py b/geest/core/algorithms/opportunities_mask_processor.py index ed64135c..114bfe56 100644 --- a/geest/core/algorithms/opportunities_mask_processor.py +++ b/geest/core/algorithms/opportunities_mask_processor.py @@ -26,6 +26,7 @@ ) from geest.core import JsonTreeItem +from geest.core.grid_column_utils import clear_grid_column, write_raster_values_to_grid from geest.utilities import log_message, resources_path from .area_iterator import AreaIterator @@ -271,8 +272,11 @@ def run(self) -> bool: bool: True if the task completed successfully, False otherwise. """ try: + # Clear stale values before writing new mask + clear_grid_column(self.study_area_gpkg_path, "opportunities_mask") + area_iterator = AreaIterator(self.study_area_gpkg_path) - for index, (current_area, clip_area, current_bbox, progress) in enumerate(area_iterator): + for index, (current_area, clip_area, current_bbox, progress, area_name) in enumerate(area_iterator): if self.feedback and self.feedback.isCanceled(): return False if self.mask_mode == "raster": @@ -292,6 +296,8 @@ def run(self) -> bool: ) if mask_layer: self.mask_list.append(mask_layer) + # Write mask values to grid column + self._write_to_grid(mask_layer, area_name) vrt_filepath = os.path.join( self.workflow_directory, @@ -632,3 +638,21 @@ def _process_raster_for_area( processing.run("gdal:rastercalculator", params) return opportunities_mask_path + + def _write_to_grid(self, raster_path: str, area_name: str) -> None: + """Write mask values to the opportunities_mask column in the grid. + + Args: + raster_path: Path to the mask raster file. + area_name: Name of the area being processed. + """ + updated = write_raster_values_to_grid( + gpkg_path=self.study_area_gpkg_path, + raster_path=raster_path, + column_name="opportunities_mask", + area_name=area_name, + ) + if updated >= 0: + log_message(f"Updated {updated} grid cells with opportunities_mask values for area {area_name}") + else: + log_message(f"Failed to write opportunities_mask values to grid for area {area_name}") diff --git a/geest/core/algorithms/population_processor.py b/geest/core/algorithms/population_processor.py index 4d5d869b..f502b70d 100644 --- a/geest/core/algorithms/population_processor.py +++ b/geest/core/algorithms/population_processor.py @@ -127,7 +127,7 @@ def clip_population_rasters(self) -> None: Clips the population raster using study area masks and records min and max values. """ area_iterator = AreaIterator(self.study_area_gpkg_path) - for index, (current_area, clip_area, current_bbox, progress) in enumerate(area_iterator): + for index, (current_area, clip_area, current_bbox, progress, area_name) in enumerate(area_iterator): if self.feedback and self.feedback.isCanceled(): return # create a temporary layer using the clip geometry @@ -257,7 +257,7 @@ def resample_population_rasters(self) -> None: area_iterator = AreaIterator(self.study_area_gpkg_path) - for index, (current_area, clip_area, current_bbox, progress) in enumerate(area_iterator): + for index, (current_area, clip_area, current_bbox, progress, area_name) in enumerate(area_iterator): if self.feedback and self.feedback.isCanceled(): return @@ -334,7 +334,7 @@ def reclassify_resampled_rasters(self) -> None: area_iterator = AreaIterator(self.study_area_gpkg_path) range_third = (self.global_max - self.global_min) / 3 - for index, (current_area, clip_area, current_bbox, progress) in enumerate(area_iterator): + for index, (current_area, clip_area, current_bbox, progress, area_name) in enumerate(area_iterator): if self.feedback and self.feedback.isCanceled(): return diff --git a/geest/core/algorithms/wee_by_population_score_processor.py b/geest/core/algorithms/wee_by_population_score_processor.py index 359df94b..7b1e4355 100644 --- a/geest/core/algorithms/wee_by_population_score_processor.py +++ b/geest/core/algorithms/wee_by_population_score_processor.py @@ -18,6 +18,7 @@ ) from geest.core.algorithms import AreaIterator +from geest.core.grid_column_utils import clear_grid_column, write_raster_values_to_grid from geest.utilities import log_message, resources_path @@ -188,9 +189,13 @@ def validate_rasters( def calculate_score(self) -> None: """ Calculates GeoE3 by POP SCORE using raster algebra and saves the result for each area. + Also writes the bivariate values to the study_area_grid column 'geoe3_by_population'. """ + # Clear stale values before writing new scores + clear_grid_column(self.study_area_gpkg_path, "geoe3_by_population") + area_iterator = AreaIterator(self.study_area_gpkg_path) - for index, (_, _, _, _) in enumerate(area_iterator): + for index, (_, _, _, _, area_name) in enumerate(area_iterator): if self.isCanceled(): return @@ -204,6 +209,8 @@ def calculate_score(self) -> None: if not self.force_clear and os.path.exists(output_path): log_message(f"Reusing existing raster: {output_path}") self.output_rasters.append(output_path) + # Still write to grid even when reusing raster + self._write_to_grid(output_path, area_name) continue log_message(f"Calculating GeoE3 by POP SCORE for area {index}") @@ -228,6 +235,27 @@ def calculate_score(self) -> None: log_message(f"GeoE3 Score raster saved to {output_path}") + # Write results to grid column + self._write_to_grid(output_path, area_name) + + def _write_to_grid(self, raster_path: str, area_name: str) -> None: + """Write bivariate score values to the geoe3_by_population column in the grid. + + Args: + raster_path: Path to the bivariate score raster file. + area_name: Name of the area being processed. + """ + updated = write_raster_values_to_grid( + gpkg_path=self.study_area_gpkg_path, + raster_path=raster_path, + column_name="geoe3_by_population", + area_name=area_name, + ) + if updated >= 0: + log_message(f"Updated {updated} grid cells with geoe3_by_population values for area {area_name}") + else: + log_message(f"Failed to write geoe3_by_population values to grid for area {area_name}") + def generate_vrt(self) -> None: """ Combines all GeoE3 Score rasters into a single VRT and applies a QML style. diff --git a/geest/core/constants.py b/geest/core/constants.py index 5e13b576..95a1e8d2 100644 --- a/geest/core/constants.py +++ b/geest/core/constants.py @@ -20,3 +20,27 @@ # Scope in QSettings APPLICATION_NAME = "GeoE3" GDAL_OUTPUT_DATA_TYPE = 6 # Float32 + +# Space2Stats defaults +DEFAULT_S2S_NTL_FIELD = "sum_viirs_ntl_2024" +DEFAULT_S2S_ENV_HAZARD_FIELDS = { + "fire": "fires_density_mean", + "flood": "pop_flood_pct", + "landslide": "landslide_susceptibility_mean_2023", + "cyclone": "cy_frequency_mean", + "drought": "drought_spei_1_5_rp100_mean", +} + +# Education proxy fields from S2S urbanization_ghssmod dataset. +# NOTE: ghs_21_pop (suburban) is intentionally excluded by design. +DEFAULT_S2S_EDUCATION_URBANIZATION_FIELDS = [ + "ghs_11_pop", + "ghs_12_pop", + "ghs_13_pop", + "ghs_22_pop", + "ghs_23_pop", + "ghs_30_pop", + "ghs_total_pop", +] + +MAX_FEATURES_FOR_VECTOR = 100000 diff --git a/geest/core/grid_column_utils.py b/geest/core/grid_column_utils.py new file mode 100644 index 00000000..745a7202 --- /dev/null +++ b/geest/core/grid_column_utils.py @@ -0,0 +1,1701 @@ +# -*- coding: utf-8 -*- +"""Grid column utilities for model-based columns. + +This module provides utilities for extracting IDs from the JSON model +and managing grid columns for indicators, factors, dimensions, and aggregate scores. + +The module supports a grid-first architecture where workflow results are written +directly to study_area_grid columns, then optionally rasterized using gdal_rasterize. +""" + +import json +import os +import re +import time +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +from osgeo import gdal, ogr +from qgis.core import Qgis, QgsFeedback, QgsVectorLayer + +from geest.utilities import log_message + + +SQLITE_WRITE_BUSY_TIMEOUT_MS = 10000 +SQLITE_WRITE_MAX_RETRIES = 3 +SQLITE_WRITE_RETRY_DELAY_SECONDS = 0.2 + + +def _open_gpkg_for_write(gpkg_path: str): + """Open GeoPackage with safer SQLite write pragmas applied.""" + ds = ogr.Open(gpkg_path, 1) + if not ds: + return None + + try: + ds.ExecuteSQL(f"PRAGMA busy_timeout={SQLITE_WRITE_BUSY_TIMEOUT_MS}") + ds.ExecuteSQL("PRAGMA journal_mode=WAL") + ds.ExecuteSQL("PRAGMA synchronous=NORMAL") + except Exception as error: + log_message(f"Failed to apply SQLite write pragmas: {error}", level=Qgis.Warning) + return ds + + +def _is_lock_error(error: Exception) -> bool: + """Return True if exception indicates SQLite lock contention.""" + text = str(error).lower() + return "database is locked" in text or "database table is locked" in text or "busy" in text + + +def _execute_sql_with_retry(ds, sql: str, dialect: Optional[str] = None): + """Execute SQL with bounded retries for lock/busy errors.""" + last_error = None + for attempt in range(SQLITE_WRITE_MAX_RETRIES): + try: + if dialect: + return ds.ExecuteSQL(sql, dialect=dialect) + return ds.ExecuteSQL(sql) + except Exception as error: + last_error = error + if _is_lock_error(error) and attempt < SQLITE_WRITE_MAX_RETRIES - 1: + time.sleep(SQLITE_WRITE_RETRY_DELAY_SECONDS * (attempt + 1)) + continue + raise + + if last_error: + raise last_error + return None + + +def _checkpoint_wal(ds) -> None: + """Force a WAL checkpoint on a GeoPackage dataset before closing. + + When multiple OGR write connections open the same GeoPackage in WAL mode, + uncheckpointed WAL data can cause QGIS's OGR provider to return stale + metadata (including empty CRS). A TRUNCATE checkpoint flushes all WAL + content back into the main database file and removes the WAL/SHM files, + ensuring subsequent readers see the full, up-to-date database. + """ + if ds is None: + return + try: + ds.ExecuteSQL("PRAGMA wal_checkpoint(TRUNCATE)") + except Exception: # nosec B110 – non-fatal; the close will still flush + pass + + +def extract_model_ids(model_path: str) -> Dict[str, List[str]]: + """Extract all IDs from the model JSON file. + + Traverses the model structure and extracts IDs for dimensions, + factors, and indicators. Prefixes are added to avoid namespace collisions: + - Dimensions: dim_ + - Factors: fac_ + - Indicators: (no prefix, most commonly referenced) + + Args: + model_path: Path to the model.json file. + + Returns: + Dictionary with keys 'dimensions', 'factors', 'indicators' containing + lists of prefixed IDs for each category. + """ + ids = { + "dimensions": [], + "factors": [], + "indicators": [], + } + + if not os.path.exists(model_path): + log_message(f"Model file not found: {model_path}", level=Qgis.Warning) + return ids + + try: + with open(model_path, "r", encoding="utf-8") as f: + model = json.load(f) + + for dimension in model.get("dimensions", []): + dim_id = dimension.get("id", "") + if dim_id: + ids["dimensions"].append(f"dim_{dim_id.lower()}") + + for factor in dimension.get("factors", []): + factor_id = factor.get("id", "") + if factor_id: + ids["factors"].append(f"fac_{factor_id.lower()}") + + for indicator in factor.get("indicators", []): + indicator_id = indicator.get("id", "") + if indicator_id: + ids["indicators"].append(indicator_id.lower()) + + except Exception as e: + log_message(f"Error extracting model IDs: {e}", level=Qgis.Critical) + + return ids + + +def get_aggregate_column_names() -> List[str]: + """Get the list of aggregate score column names. + + Returns: + List of column names for aggregate scores (WEE score, WEE by population, etc.) + """ + return [ + "geoe3", + "geoe3_by_population", # GeoE3 × Population bivariate score (1-15) + "geoe3_masked", # GeoE3 score masked by opportunities/GHSL + "geoe3_by_population_masked", # GeoE3 by population masked by opportunities + "opportunities_mask", # Binary mask for job opportunities + "contextual_score", + "accessibility_score", + "place_characterization_score", + ] + + +def get_all_column_names(model_path: str) -> List[str]: + """Get all column names to be added to the grid layer. + + Args: + model_path: Path to the model.json file. + + Returns: + List of all column names (indicators, factors, dimensions, and aggregates). + """ + ids = extract_model_ids(model_path) + columns = [] + + # Add indicator columns + columns.extend(ids["indicators"]) + + # Add factor columns + columns.extend(ids["factors"]) + + # Add dimension columns + columns.extend(ids["dimensions"]) + + # Add aggregate columns + columns.extend(get_aggregate_column_names()) + + return columns + + +def add_model_columns_to_grid(gpkg_path: str, model_path: str) -> bool: + """Add model-based columns to the study_area_grid layer. + + Adds one Real/Float column for each indicator, factor, dimension, and aggregate + score based on the IDs from the model.json file. + + Args: + gpkg_path: Path to the GeoPackage containing study_area_grid. + model_path: Path to the model.json file. + + Returns: + True if columns were added successfully, False otherwise. + """ + column_names = get_all_column_names(model_path) + + if not column_names: + log_message("No columns to add to grid layer", level=Qgis.Warning) + return False + + try: + ds = _open_gpkg_for_write(gpkg_path) + if not ds: + log_message(f"Could not open GeoPackage: {gpkg_path}", level=Qgis.Critical) + return False + + layer = ds.GetLayerByName("study_area_grid") + if not layer: + log_message("study_area_grid layer not found", level=Qgis.Critical) + ds = None + return False + + # Get existing field names to avoid duplicates + layer_defn = layer.GetLayerDefn() + existing_fields = set() + for i in range(layer_defn.GetFieldCount()): + existing_fields.add(layer_defn.GetFieldDefn(i).GetName().lower()) + + # Add new columns as Real/Float type + added_count = 0 + for col_name in column_names: + # Sanitize column name (replace spaces with underscores, limit length) + sanitized_name = col_name.replace(" ", "_").replace("-", "_")[:63] + + if sanitized_name.lower() in existing_fields: + continue + + field_defn = ogr.FieldDefn(sanitized_name, ogr.OFTReal) + if layer.CreateField(field_defn) != 0: + log_message(f"Failed to create field: {sanitized_name}", level=Qgis.Warning) + else: + added_count += 1 + + ds.FlushCache() + ds = None + + log_message(f"Added {added_count} model columns to study_area_grid") + return True + + except Exception as e: + log_message(f"Error adding model columns to grid: {e}", level=Qgis.Critical) + return False + + +def write_raster_values_to_grid( + gpkg_path: str, + raster_path: str, + column_name: str, + area_name: Optional[str] = None, +) -> int: + """Sample raster values at grid cell centroids and write to grid column. + + Uses the raster's extent to spatially filter grid cells, then samples + only those cells that fall within the raster bounds. Skips nodata values. + + Args: + gpkg_path: Path to the GeoPackage containing study_area_grid. + raster_path: Path to the raster file to sample. + column_name: Name of the column to write values to. + area_name: Optional area name to filter grid cells. If None, processes all cells. + + Returns: + Number of cells updated, or -1 on error. + """ + if not os.path.exists(raster_path): + log_message(f"Raster file not found: {raster_path}", level=Qgis.Warning) + return -1 + + try: + # Open the raster + raster_ds = gdal.Open(raster_path) + if not raster_ds: + log_message(f"Could not open raster: {raster_path}", level=Qgis.Critical) + return -1 + + band = raster_ds.GetRasterBand(1) + nodata = band.GetNoDataValue() + gt = raster_ds.GetGeoTransform() + + # Calculate raster extent for spatial filtering + xmin = gt[0] + ymax = gt[3] + xmax = gt[0] + gt[1] * raster_ds.RasterXSize + ymin = gt[3] + gt[5] * raster_ds.RasterYSize + + # Open the GeoPackage for updating + ds = _open_gpkg_for_write(gpkg_path) + if not ds: + log_message(f"Could not open GeoPackage: {gpkg_path}", level=Qgis.Critical) + raster_ds = None + return -1 + + # Get layer and ensure column exists (create if missing) + layer, field_idx = _get_grid_layer_and_field_index(ds, column_name, create_if_missing=True) + if layer is None or field_idx < 0: + ds = None + raster_ds = None + return -1 + + sanitized_column = _sanitize_column_name(column_name) + + # Set spatial filter to raster extent (only process cells within raster bounds) + layer.SetSpatialFilterRect(xmin, ymin, xmax, ymax) + + # Set attribute filter if area_name is provided + if area_name: + layer.SetAttributeFilter(f"area_name = '{area_name}'") + + # Collect FIDs and values first, then batch update + fid_values = {} + for feature in layer: + geom = feature.GetGeometryRef() + if not geom: + continue + + # Get centroid + centroid = geom.Centroid() + x = centroid.GetX() + y = centroid.GetY() + + # Convert to pixel coordinates + px = int((x - gt[0]) / gt[1]) + py = int((y - gt[3]) / gt[5]) + + # Check bounds (should be within due to spatial filter, but double-check) + if px < 0 or px >= raster_ds.RasterXSize or py < 0 or py >= raster_ds.RasterYSize: + continue + + # Read pixel value + try: + pixel_value = band.ReadAsArray(px, py, 1, 1) + if pixel_value is not None: + value = float(pixel_value[0, 0]) + # Skip nodata values + if nodata is not None and value == nodata: + continue + fid_values[feature.GetFID()] = value + except (RuntimeError, ValueError, IndexError): + # Skip cells where pixel read fails + continue + + log_message(f"Found {len(fid_values)} grid cells with valid raster values") + + # Reset filters before updating + layer.SetSpatialFilter(None) + layer.SetAttributeFilter(None) + layer.ResetReading() + + # Batch update using SQL for efficiency + updated_count = 0 + batch_size = 500 + fids = list(fid_values.keys()) + + for batch_start in range(0, len(fids), batch_size): + batch_fids = fids[batch_start : batch_start + batch_size] + + # Build CASE statement for this batch + case_parts = [] + for fid in batch_fids: + value = fid_values[fid] + case_parts.append(f"WHEN fid = {fid} THEN {value}") + + if case_parts: + fid_list = ",".join(str(f) for f in batch_fids) + sql = ( + f"UPDATE study_area_grid " # nosec B608 + f'SET "{sanitized_column}" = CASE {" ".join(case_parts)} END ' + f"WHERE fid IN ({fid_list})" + ) + _execute_sql_with_retry(ds, sql) + updated_count += len(batch_fids) + + ds = None + raster_ds = None + + log_message(f"Updated {updated_count} grid cells for column {sanitized_column}") + return updated_count + + except Exception as e: + log_message(f"Error in write_raster_values_to_grid: {e}", level=Qgis.Critical) + return -1 + + +def _sanitize_column_name(column_name: str) -> str: + """Sanitize a column name for use in SQL and as a field name. + + Args: + column_name: The column name to sanitize. + + Returns: + Sanitized column name (lowercase, underscores, max 63 chars). + """ + return column_name.replace(" ", "_").replace("-", "_")[:63].lower() + + +def _quote_sql_identifier(identifier: str) -> str: + """Quote and validate an SQL identifier for SQLite usage. + + Args: + identifier: The identifier to validate and quote. + + Returns: + Safely quoted identifier string. + + Raises: + ValueError: If identifier contains unsupported characters. + """ + if not identifier: + raise ValueError("SQL identifier cannot be empty") + + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", identifier): + raise ValueError(f"Invalid SQL identifier: {identifier}") + + return f'"{identifier}"' + + +def _quote_sql_literal(value: str) -> str: + """Quote a string literal for SQLite usage.""" + return "'" + value.replace("'", "''") + "'" + + +def _parse_reclass_boundary(boundary: Any) -> float: + """Parse a reclassification table boundary value into float.""" + if isinstance(boundary, str): + token = boundary.strip().lower() + if token in {"inf", "+inf", "infinity", "+infinity"}: + return float("inf") + if token in {"-inf", "-infinity"}: + return float("-inf") + return float(token) + return float(boundary) + + +def _value_matches_range(value: float, minimum: float, maximum: float, range_boundaries: int) -> bool: + """Check whether a value falls into a reclassification range.""" + if range_boundaries == 0: + return minimum < value <= maximum + if range_boundaries == 1: + return minimum <= value < maximum + if range_boundaries == 2: + return minimum <= value <= maximum + if range_boundaries == 3: + return minimum < value < maximum + return minimum < value <= maximum + + +def get_grid_column_values(gpkg_path: str, column_name: str, area_name: Optional[str] = None) -> List[float]: + """Read non-null values from a grid column, optionally filtered by area.""" + if not os.path.exists(gpkg_path): + return [] + + values: List[float] = [] + try: + ds = ogr.Open(gpkg_path, 0) + if not ds: + return values + + layer, field_idx = _get_grid_layer_and_field_index(ds, column_name, create_if_missing=False) + if layer is None or field_idx < 0: + ds = None + return values + + sanitized_column = _sanitize_column_name(column_name) + if area_name: + layer.SetAttributeFilter(f"area_name = '{area_name}'") + + for feature in layer: + value = feature.GetField(sanitized_column) + if value is None: + continue + try: + values.append(float(value)) + except (TypeError, ValueError): + continue + + layer.SetAttributeFilter(None) + ds = None + return values + except Exception as error: + log_message(f"Error reading grid column values: {error}", level=Qgis.Warning) + return [] + + +def reclassify_grid_column_with_table( + gpkg_path: str, + column_name: str, + reclassification_table: List[Any], + area_name: Optional[str] = None, + range_boundaries: int = 0, +) -> int: + """Apply a QGIS-style reclassification table directly to a grid column. + + The table format must be [min1, max1, class1, min2, max2, class2, ...]. + """ + if len(reclassification_table) % 3 != 0: + log_message("Invalid reclassification table length", level=Qgis.Warning) + return -1 + + if not os.path.exists(gpkg_path): + log_message(f"GeoPackage not found: {gpkg_path}", level=Qgis.Warning) + return -1 + + try: + parsed_ranges: List[Tuple[float, float, float]] = [] + for index in range(0, len(reclassification_table), 3): + minimum = _parse_reclass_boundary(reclassification_table[index]) + maximum = _parse_reclass_boundary(reclassification_table[index + 1]) + output_value = float(reclassification_table[index + 2]) + parsed_ranges.append((minimum, maximum, output_value)) + except Exception as error: + log_message(f"Failed to parse reclassification table: {error}", level=Qgis.Critical) + return -1 + + try: + ds = _open_gpkg_for_write(gpkg_path) + if not ds: + log_message(f"Could not open GeoPackage: {gpkg_path}", level=Qgis.Critical) + return -1 + + layer, field_idx = _get_grid_layer_and_field_index(ds, column_name, create_if_missing=False) + if layer is None or field_idx < 0: + ds = None + return -1 + + sanitized_column = _sanitize_column_name(column_name) + if area_name: + layer.SetAttributeFilter(f"area_name = '{area_name}'") + + updated_count = 0 + layer.StartTransaction() + try: + for feature in layer: + value = feature.GetField(sanitized_column) + if value is None: + continue + + numeric_value = float(value) + mapped_value = None + for minimum, maximum, output_value in parsed_ranges: + if _value_matches_range(numeric_value, minimum, maximum, range_boundaries): + mapped_value = output_value + break + + if mapped_value is None: + continue + + feature.SetField(sanitized_column, mapped_value) + if layer.SetFeature(feature) == 0: + updated_count += 1 + + layer.CommitTransaction() + except Exception: + layer.RollbackTransaction() + raise + + layer.SetAttributeFilter(None) + ds = None + return updated_count + except Exception as error: + log_message(f"Error reclassifying grid column: {error}", level=Qgis.Critical) + return -1 + + +def _get_grid_layer_and_field_index( + ds: ogr.DataSource, + column_name: str, + create_if_missing: bool = True, +) -> Tuple[Optional[ogr.Layer], int]: + """Get the study_area_grid layer and field index for a column. + + Args: + ds: Open OGR DataSource for the GeoPackage. + column_name: The column name to look up. + create_if_missing: If True, create the column as Real/Float if it doesn't exist. + + Returns: + Tuple of (layer, field_index) or (None, -1) if not found. + """ + layer = ds.GetLayerByName("study_area_grid") + if not layer: + log_message("study_area_grid layer not found", level=Qgis.Critical) + return None, -1 + + sanitized_column = _sanitize_column_name(column_name) + layer_defn = layer.GetLayerDefn() + field_idx = layer_defn.GetFieldIndex(sanitized_column) + + if field_idx < 0: + if create_if_missing: + # Create the column as Real/Float type + field_defn = ogr.FieldDefn(sanitized_column, ogr.OFTReal) + if layer.CreateField(field_defn) != 0: + log_message(f"Failed to create column {sanitized_column}", level=Qgis.Warning) + return layer, -1 + log_message(f"Created column {sanitized_column} in grid layer") + # Re-fetch the field index after creation + layer_defn = layer.GetLayerDefn() + field_idx = layer_defn.GetFieldIndex(sanitized_column) + else: + log_message(f"Column {sanitized_column} not found in grid layer", level=Qgis.Warning) + return layer, -1 + + return layer, field_idx + + +def write_joined_values_to_grid( + gpkg_path: str, + column_name: str, + source_gpkg: str, + source_layer: str, + source_key_field: str, + target_key_field: str, + source_value_field: str, + area_name: Optional[str] = None, +) -> int: + """Write values to study_area_grid via key-based join. + + This function joins `study_area_grid` in the target GeoPackage with an external + source layer and writes matched values to a target grid column. + + Typical usage for regional S2S: + - target_key_field: h3_index + - source_key_field: hex_id + + Args: + gpkg_path: Path to the GeoPackage containing study_area_grid. + column_name: Target grid column to write values into. + source_gpkg: Path to source GeoPackage containing source_layer. + source_layer: Source layer name in source_gpkg. + source_key_field: Source key field name (e.g. hex_id). + target_key_field: Grid key field name (e.g. h3_index). + source_value_field: Source value field name to copy. + area_name: Optional area_name filter for grid rows. + + Returns: + Number of matched grid rows updated, or -1 on error. + """ + if not os.path.exists(gpkg_path): + log_message(f"GeoPackage not found: {gpkg_path}", level=Qgis.Warning) + return -1 + + if not os.path.exists(source_gpkg): + log_message(f"Source GeoPackage not found: {source_gpkg}", level=Qgis.Warning) + return -1 + + try: + # Ensure the target column exists in study_area_grid. + ds = _open_gpkg_for_write(gpkg_path) + if not ds: + log_message(f"Could not open GeoPackage: {gpkg_path}", level=Qgis.Critical) + return -1 + + layer, field_idx = _get_grid_layer_and_field_index(ds, column_name, create_if_missing=True) + ds = None + if layer is None or field_idx < 0: + return -1 + + sanitized_column = _sanitize_column_name(column_name) + + target_col_sql = _quote_sql_identifier(sanitized_column) + target_key_sql = _quote_sql_identifier(target_key_field) + source_key_sql = _quote_sql_identifier(source_key_field) + source_value_sql = _quote_sql_identifier(source_value_field) + source_layer_sql = _quote_sql_identifier(source_layer) + + area_predicate = "" + if area_name: + area_predicate = f"AND g.area_name = {_quote_sql_literal(area_name)}" + + source_gpkg_literal = _quote_sql_literal(source_gpkg) + source_layer_literal = _quote_sql_literal(source_layer) + + ds = _open_gpkg_for_write(gpkg_path) + if not ds: + log_message(f"Could not open GeoPackage: {gpkg_path}", level=Qgis.Critical) + return -1 + + _execute_sql_with_retry(ds, f"ATTACH DATABASE {source_gpkg_literal} AS src", dialect="SQLite") # nosec B608 + + try: + # Validate source layer exists. + source_exists_result = _execute_sql_with_retry( + ds, + ( + "SELECT 1 AS exists_flag " # nosec B608 + "FROM src.sqlite_master " + "WHERE type IN ('table', 'view') " + f"AND name = {source_layer_literal} " + "LIMIT 1" + ), + dialect="SQLite", + ) + source_exists = False + if source_exists_result is not None: + feature = source_exists_result.GetNextFeature() + source_exists = feature is not None + ds.ReleaseResultSet(source_exists_result) + + if not source_exists: + log_message(f"Source layer not found in source GeoPackage: {source_layer}", level=Qgis.Warning) + return -1 + + # Clear existing values to preserve NULL semantics for unmatched keys. + clear_sql = f"UPDATE study_area_grid SET {target_col_sql} = NULL" # nosec B608 + if area_name: + clear_sql += f" WHERE area_name = {_quote_sql_literal(area_name)}" + _execute_sql_with_retry(ds, clear_sql, dialect="SQLite") # nosec B608 + + update_sql = ( + f"UPDATE study_area_grid AS g " # nosec B608 + f"SET {target_col_sql} = (" + f"SELECT CAST(s.{source_value_sql} AS REAL) " + f"FROM src.{source_layer_sql} AS s " + f"WHERE s.{source_key_sql} = g.{target_key_sql} " + f"LIMIT 1" + f") " + f"WHERE EXISTS (" + f"SELECT 1 FROM src.{source_layer_sql} AS s " + f"WHERE s.{source_key_sql} = g.{target_key_sql}" + f") {area_predicate}" + ) + _execute_sql_with_retry(ds, update_sql, dialect="SQLite") # nosec B608 + + count_sql = ( + f"SELECT COUNT(*) AS matched_count " # nosec B608 + f"FROM study_area_grid AS g " + f"JOIN src.{source_layer_sql} AS s " + f"ON s.{source_key_sql} = g.{target_key_sql} " + f"WHERE 1=1 {area_predicate}" + ) + count_result = _execute_sql_with_retry(ds, count_sql, dialect="SQLite") + matched_count = 0 + if count_result is not None: + feature = count_result.GetNextFeature() + if feature is not None: + matched_count = feature.GetField("matched_count") or 0 + ds.ReleaseResultSet(count_result) + finally: + _execute_sql_with_retry(ds, "DETACH DATABASE src", dialect="SQLite") + ds = None + + log_message( + f"Updated {matched_count} grid rows in {sanitized_column} using key join " + f"({target_key_field} <- {source_key_field})" + ) + return int(matched_count) + + except Exception as e: + log_message(f"Error in write_joined_values_to_grid: {e}", level=Qgis.Critical) + return -1 + + +def write_uniform_value_to_grid( + gpkg_path: str, + column_name: str, + value: float, + area_name: Optional[str] = None, + clip_geometry: Optional[ogr.Geometry] = None, +) -> int: + """Write a constant value to all cells in an area using SQL UPDATE. + + This is useful for index_score workflows where a single value applies + to all grid cells in an area. + + Args: + gpkg_path: Path to the GeoPackage containing study_area_grid. + column_name: Name of the column to write values to. + value: The constant value to write to all matching cells. + area_name: Optional area name to filter grid cells. + clip_geometry: Optional geometry to spatially filter cells (not used in SQL mode). + + Returns: + Number of cells updated, or -1 on error. + """ + _ = clip_geometry # Not used in SQL mode + + if not os.path.exists(gpkg_path): + log_message(f"GeoPackage not found: {gpkg_path}", level=Qgis.Warning) + return -1 + + sanitized_column = _sanitize_column_name(column_name) + + try: + ds = _open_gpkg_for_write(gpkg_path) + if not ds: + log_message(f"Could not open GeoPackage: {gpkg_path}", level=Qgis.Critical) + return -1 + + # Verify column exists, create if missing + layer, field_idx = _get_grid_layer_and_field_index(ds, column_name, create_if_missing=True) + if layer is None or field_idx < 0: + ds = None + return -1 + + # Simple SQL UPDATE - no area_name filter, update ALL cells + sql = f'UPDATE study_area_grid SET "{sanitized_column}" = {value}' # nosec B608 + log_message(f"Executing: {sql}") + _execute_sql_with_retry(ds, sql) # nosec B608 + ds = None + + return 0 + + except Exception as e: + log_message(f"Error in write_uniform_value_to_grid: {e}", level=Qgis.Critical) + return -1 + + +def clear_grid_column(gpkg_path: str, column_name: str) -> bool: + """Set all values in a grid column to NULL. + + Should be called before populating a column to ensure clean state. + + Args: + gpkg_path: Path to the GeoPackage containing study_area_grid. + column_name: Name of the column to clear. + + Returns: + True if successful, False otherwise. + """ + if not os.path.exists(gpkg_path): + log_message(f"GeoPackage not found: {gpkg_path}", level=Qgis.Warning) + return False + + sanitized_column = _sanitize_column_name(column_name) + + try: + ds = _open_gpkg_for_write(gpkg_path) + if not ds: + log_message(f"Could not open GeoPackage: {gpkg_path}", level=Qgis.Critical) + return False + + sql = f'UPDATE study_area_grid SET "{sanitized_column}" = NULL' # nosec B608 + log_message(f"Clearing column: {sql}") + _execute_sql_with_retry(ds, sql) # nosec B608 + ds = None + return True + + except Exception as e: + log_message(f"Error clearing grid column: {e}", level=Qgis.Critical) + return False + + +def count_features_per_grid_cell( + gpkg_path: str, + column_name: str, + features_layer: QgsVectorLayer, + feedback: QgsFeedback = None, +) -> int: + """Count features intersecting each grid cell and assign scores. + + Writes directly to study_area_grid without creating copies. + Score mapping: 0 features = NULL, 1 feature = 3, 2+ features = 5 + + Args: + gpkg_path: Path to the GeoPackage containing study_area_grid. + column_name: Name of the column to write values to. + features_layer: QgsVectorLayer containing features to count. + feedback: Optional feedback for progress reporting. + + Returns: + Number of cells updated, or -1 on error. + """ + from qgis.core import QgsFeatureRequest, QgsSpatialIndex + + if not os.path.exists(gpkg_path): + log_message(f"GeoPackage not found: {gpkg_path}", level=Qgis.Warning) + return -1 + + sanitized_column = _sanitize_column_name(column_name) + + try: + # Load grid layer + grid_layer = QgsVectorLayer(f"{gpkg_path}|layername=study_area_grid", "grid", "ogr") + if not grid_layer.isValid(): + log_message("Could not load study_area_grid layer", level=Qgis.Critical) + return -1 + + # Create spatial index for grid + grid_index = QgsSpatialIndex(grid_layer.getFeatures()) + + # Count features per cell + grid_feature_counts = {} + feature_count = features_layer.featureCount() + log_message(f"Counting {feature_count} features against grid cells") + + for i, feature in enumerate(features_layer.getFeatures()): + geom = feature.geometry() + if geom.isEmpty(): + continue + + # Find intersecting grid cells + intersecting_ids = grid_index.intersects(geom.boundingBox()) + + # Refine with actual intersection test for non-point geometries + if geom.type() != 0: # Not point + request = QgsFeatureRequest().setFilterFids(intersecting_ids) + intersecting_ids = [f.id() for f in grid_layer.getFeatures(request) if f.geometry().intersects(geom)] + + for grid_id in intersecting_ids: + grid_feature_counts[grid_id] = grid_feature_counts.get(grid_id, 0) + 1 + + if feedback and i % 1000 == 0: + feedback.setProgress((i / feature_count) * 50) + + log_message(f"Found {len(grid_feature_counts)} grid cells with features") + + # Build SQL CASE statement for batch update + ds = _open_gpkg_for_write(gpkg_path) + if not ds: + log_message(f"Could not open GeoPackage: {gpkg_path}", level=Qgis.Critical) + return -1 + + # Update in batches using SQL + # First, get the fid field name (usually 'fid' for GeoPackage) + updated_count = 0 + batch_size = 500 + fids = list(grid_feature_counts.keys()) + + for batch_start in range(0, len(fids), batch_size): + batch_fids = fids[batch_start : batch_start + batch_size] + + # Build CASE statement for this batch + case_parts = [] + for fid in batch_fids: + count = grid_feature_counts[fid] + score = 3 if count == 1 else 5 + case_parts.append(f"WHEN fid = {fid} THEN {score}") + + if case_parts: + fid_list = ",".join(str(f) for f in batch_fids) + sql = ( + f"UPDATE study_area_grid " # nosec B608 + f'SET "{sanitized_column}" = CASE {" ".join(case_parts)} END ' + f"WHERE fid IN ({fid_list})" + ) + _execute_sql_with_retry(ds, sql) + updated_count += len(batch_fids) + + if feedback: + progress = 50 + (batch_start / len(fids)) * 50 + feedback.setProgress(progress) + + ds = None + log_message(f"Updated {updated_count} grid cells with feature counts") + return updated_count + + except Exception as e: + log_message(f"Error in count_features_per_grid_cell: {e}", level=Qgis.Critical) + return -1 + + +def write_spatial_join_to_grid( + gpkg_path: str, + column_name: str, + features_gpkg: str, + features_layer: str, + score_expression: Union[str, Callable[[ogr.Feature], float]], + area_name: Optional[str] = None, + aggregation_method: str = "MAX", + save_buffers: bool = True, + workflow_directory: Optional[str] = None, +) -> int: + """Write scores to grid cells based on spatial intersection with features. + + This function performs a spatial join between grid cells and input features, + applying an aggregation method (MAX, MIN, AVG, SUM) to determine the final + score for each cell. + + Args: + gpkg_path: Path to the GeoPackage containing study_area_grid. + column_name: Name of the column to write values to. + features_gpkg: Path to the GeoPackage containing the features to join. + features_layer: Name of the layer containing features (e.g., buffer polygons). + score_expression: Either a field name containing scores, or a callable + that takes a feature and returns a score. + area_name: Optional area name to filter grid cells. + aggregation_method: How to combine multiple intersecting features + (MAX, MIN, AVG, SUM, COUNT). Defaults to MAX. + save_buffers: If True, save intermediate buffer table for review. + workflow_directory: Directory to save intermediate files. + + Returns: + Number of cells updated, or -1 on error. + """ + if not os.path.exists(gpkg_path): + log_message(f"GeoPackage not found: {gpkg_path}", level=Qgis.Warning) + return -1 + + if not os.path.exists(features_gpkg): + log_message(f"Features GeoPackage not found: {features_gpkg}", level=Qgis.Warning) + return -1 + + try: + # Open the main GeoPackage for updating + ds = _open_gpkg_for_write(gpkg_path) + if not ds: + log_message(f"Could not open GeoPackage: {gpkg_path}", level=Qgis.Critical) + return -1 + + layer, field_idx = _get_grid_layer_and_field_index(ds, column_name) + if layer is None or field_idx < 0: + ds = None + return -1 + + sanitized_column = _sanitize_column_name(column_name) + + # Open the features GeoPackage + features_ds = ogr.Open(features_gpkg, 0) + if not features_ds: + log_message(f"Could not open features GeoPackage: {features_gpkg}", level=Qgis.Critical) + ds = None + return -1 + + features_lyr = features_ds.GetLayerByName(features_layer) + if not features_lyr: + log_message(f"Features layer not found: {features_layer}", level=Qgis.Critical) + features_ds = None + ds = None + return -1 + + # Build spatial index for features if not already indexed + # Note: GeoPackage layers should have spatial index by default + + # Set attribute filter on grid layer + if area_name: + layer.SetAttributeFilter(f"area_name = '{area_name}'") + + # First pass: collect FIDs and compute scores + fid_scores = {} + for grid_feature in layer: + grid_geom = grid_feature.GetGeometryRef() + if not grid_geom: + continue + + fid = grid_feature.GetFID() + + # Find intersecting features + features_lyr.SetSpatialFilter(grid_geom) + scores = [] + + for feat in features_lyr: + feat_geom = feat.GetGeometryRef() + if feat_geom and grid_geom.Intersects(feat_geom): + # Get score from expression + if callable(score_expression): + score = score_expression(feat) + else: + # It's a field name + score = feat.GetField(score_expression) + + if score is not None: + scores.append(float(score)) + + features_lyr.SetSpatialFilter(None) + + # Aggregate scores + if scores: + if aggregation_method == "MAX": + final_score = max(scores) + elif aggregation_method == "MIN": + final_score = min(scores) + elif aggregation_method == "AVG": + final_score = sum(scores) / len(scores) + elif aggregation_method == "SUM": + final_score = sum(scores) + elif aggregation_method == "COUNT": + final_score = float(len(scores)) + else: + final_score = max(scores) + + fid_scores[fid] = final_score + + log_message(f"Found {len(fid_scores)} grid cells with intersecting features for spatial join") + + # Reset filter before updating + layer.SetAttributeFilter(None) + layer.ResetReading() + + # Second pass: update features by FID + updated_count = 0 + layer.StartTransaction() + + try: + for fid, score in fid_scores.items(): + feature = layer.GetFeature(fid) + if feature: + feature.SetField(sanitized_column, score) + if layer.SetFeature(feature) == 0: + updated_count += 1 + + layer.CommitTransaction() + + except Exception as e: + layer.RollbackTransaction() + log_message(f"Error in spatial join: {e}", level=Qgis.Critical) + features_ds = None + ds = None + return -1 + + # Save intermediate buffers if requested + if save_buffers and workflow_directory: + buffer_output = os.path.join(workflow_directory, f"{features_layer}_buffers.gpkg") + try: + driver = ogr.GetDriverByName("GPKG") + if os.path.exists(buffer_output): + driver.DeleteDataSource(buffer_output) + buffer_ds = driver.CreateDataSource(buffer_output) + buffer_ds.CopyLayer(features_lyr, features_layer) + buffer_ds = None + log_message(f"Saved intermediate buffers to {buffer_output}") + except Exception as e: + log_message(f"Could not save intermediate buffers: {e}", level=Qgis.Warning) + + features_ds = None + ds.FlushCache() + ds = None + + log_message(f"Updated {updated_count} grid cells via spatial join for column {sanitized_column}") + return updated_count + + except Exception as e: + log_message(f"Error in write_spatial_join_to_grid: {e}", level=Qgis.Critical) + return -1 + + +def write_point_count_to_grid( + gpkg_path: str, + column_name: str, + features_gpkg: str, + features_layer: str, + area_name: Optional[str] = None, + count_to_score_mapping: Optional[Dict[int, float]] = None, + max_count_score: float = 5.0, +) -> int: + """Count points per grid cell and map counts to scores. + + This function counts point features within each grid cell and converts + the count to a score using the provided mapping or a default linear scale. + + Args: + gpkg_path: Path to the GeoPackage containing study_area_grid. + column_name: Name of the column to write values to. + features_gpkg: Path to the GeoPackage containing the point features. + features_layer: Name of the layer containing point features. + area_name: Optional area name to filter grid cells. + count_to_score_mapping: Dict mapping count values to scores. + Example: {0: 0, 1: 3, 2: 5} means 0 points = score 0, + 1 point = score 3, 2+ points = score 5. + If None, uses default {0: 0, 1: 3} with max as 5. + max_count_score: Score to assign when count exceeds all mappings. + + Returns: + Number of cells updated, or -1 on error. + """ + # Default mapping based on typical GeoE3 point per cell scoring + if count_to_score_mapping is None: + count_to_score_mapping = {0: 0, 1: 3} + + if not os.path.exists(gpkg_path): + log_message(f"GeoPackage not found: {gpkg_path}", level=Qgis.Warning) + return -1 + + if not os.path.exists(features_gpkg): + log_message(f"Features GeoPackage not found: {features_gpkg}", level=Qgis.Warning) + return -1 + + try: + # Open the main GeoPackage for updating + ds = _open_gpkg_for_write(gpkg_path) + if not ds: + log_message(f"Could not open GeoPackage: {gpkg_path}", level=Qgis.Critical) + return -1 + + layer, field_idx = _get_grid_layer_and_field_index(ds, column_name) + if layer is None or field_idx < 0: + ds = None + return -1 + + sanitized_column = _sanitize_column_name(column_name) + + # Open the features GeoPackage + features_ds = ogr.Open(features_gpkg, 0) + if not features_ds: + log_message(f"Could not open features GeoPackage: {features_gpkg}", level=Qgis.Critical) + ds = None + return -1 + + features_lyr = features_ds.GetLayerByName(features_layer) + if not features_lyr: + log_message(f"Features layer not found: {features_layer}", level=Qgis.Critical) + features_ds = None + ds = None + return -1 + + # Set attribute filter on grid layer + if area_name: + layer.SetAttributeFilter(f"area_name = '{area_name}'") + + # Get sorted mapping keys for lookup + sorted_counts = sorted(count_to_score_mapping.keys()) + + # First pass: collect FIDs and compute scores + fid_scores = {} + for grid_feature in layer: + grid_geom = grid_feature.GetGeometryRef() + if not grid_geom: + continue + + fid = grid_feature.GetFID() + + # Count intersecting features + features_lyr.SetSpatialFilter(grid_geom) + point_count = 0 + + for feat in features_lyr: + feat_geom = feat.GetGeometryRef() + if feat_geom and grid_geom.Intersects(feat_geom): + point_count += 1 + + features_lyr.SetSpatialFilter(None) + + # Map count to score + score = max_count_score # Default to max if count exceeds all mappings + for count_threshold in sorted_counts: + if point_count <= count_threshold: + score = count_to_score_mapping[count_threshold] + break + + # If point_count exceeds all thresholds, use max_count_score + if point_count > max(sorted_counts): + score = max_count_score + + fid_scores[fid] = score + + log_message(f"Found {len(fid_scores)} grid cells to update with point counts") + + # Reset filter before updating + layer.SetAttributeFilter(None) + layer.ResetReading() + + # Second pass: update features by FID + updated_count = 0 + layer.StartTransaction() + + try: + for fid, score in fid_scores.items(): + feature = layer.GetFeature(fid) + if feature: + feature.SetField(sanitized_column, score) + if layer.SetFeature(feature) == 0: + updated_count += 1 + + layer.CommitTransaction() + + except Exception as e: + layer.RollbackTransaction() + log_message(f"Error in point count: {e}", level=Qgis.Critical) + features_ds = None + ds = None + return -1 + + features_ds = None + ds.FlushCache() + ds = None + + log_message(f"Updated {updated_count} grid cells with point counts for column {sanitized_column}") + return updated_count + + except Exception as e: + log_message(f"Error in write_point_count_to_grid: {e}", level=Qgis.Critical) + return -1 + + +def write_aggregation_to_grid( + gpkg_path: str, + target_column: str, + source_columns_weights: Dict[str, float], + area_name: Optional[str] = None, + use_coalesce: bool = True, +) -> int: + """Perform weighted aggregation of grid columns using SQL UPDATE. + + This replaces the raster-based QgsRasterCalculator approach for + factor, dimension, and analysis aggregations. + + Uses a single SQL UPDATE statement: + UPDATE study_area_grid SET target = (w1*COALESCE(c1,0) + w2*COALESCE(c2,0) + ...) + + Args: + gpkg_path: Path to the GeoPackage containing study_area_grid. + target_column: Name of the column to write aggregated values to. + source_columns_weights: Dict mapping source column names to their weights. + Example: {"indicator1": 0.3, "indicator2": 0.3, "indicator3": 0.4} + area_name: Optional area name to filter grid cells (not used - aggregates all). + use_coalesce: If True, use COALESCE(col, 0) to handle NULL values. + Defaults to True. + + Returns: + 0 on success, or -1 on error. + """ + _ = area_name # Not used - we aggregate all cells + + if not source_columns_weights: + log_message("No source columns provided for aggregation", level=Qgis.Warning) + return -1 + + if not os.path.exists(gpkg_path): + log_message(f"GeoPackage not found: {gpkg_path}", level=Qgis.Warning) + return -1 + + try: + ds = _open_gpkg_for_write(gpkg_path) + if not ds: + log_message(f"Could not open GeoPackage: {gpkg_path}", level=Qgis.Critical) + return -1 + + layer, field_idx = _get_grid_layer_and_field_index(ds, target_column) + if layer is None or field_idx < 0: + ds = None + return -1 + + sanitized_target = _sanitize_column_name(target_column) + + # Verify all source columns exist + layer_defn = layer.GetLayerDefn() + created_missing_columns = [] + for source_col in source_columns_weights.keys(): + sanitized_source = _sanitize_column_name(source_col) + if layer_defn.GetFieldIndex(sanitized_source) < 0: + field_defn = ogr.FieldDefn(sanitized_source, ogr.OFTReal) + if layer.CreateField(field_defn) != 0: + log_message(f"Source column {sanitized_source} not found in grid layer", level=Qgis.Warning) + ds = None + return -1 + created_missing_columns.append(sanitized_source) + layer_defn = layer.GetLayerDefn() + + if created_missing_columns: + log_message( + f"Created missing aggregation source columns with NULL defaults: {', '.join(created_missing_columns)}", + level=Qgis.Warning, + ) + + # Build the weighted sum expression + # Example: (0.3 * COALESCE("indicator1", 0) + 0.4 * COALESCE("indicator2", 0)) + terms = [] + for source_col, weight in source_columns_weights.items(): + sanitized_source = _sanitize_column_name(source_col) + if use_coalesce: + terms.append(f'({weight} * COALESCE("{sanitized_source}", 0))') + else: + terms.append(f'({weight} * "{sanitized_source}")') + + expression = " + ".join(terms) + + # Build and execute SQL UPDATE + sql = f'UPDATE study_area_grid SET "{sanitized_target}" = ({expression})' # nosec B608 + log_message(f"Executing aggregation SQL: {sql[:200]}...") + _execute_sql_with_retry(ds, sql) # nosec B608 + ds = None + + log_message(f"Aggregated {len(source_columns_weights)} columns into {sanitized_target}") + return 0 + + except Exception as e: + log_message(f"Error in write_aggregation_to_grid: {e}", level=Qgis.Critical) + return -1 + + +def rasterize_grid_column( + gpkg_path: str, + column_name: str, + output_raster_path: str, + cell_size: float, + extent: Optional[Tuple[float, float, float, float]] = None, + nodata: float = -9999.0, + area_name: Optional[str] = None, + output_type: int = gdal.GDT_Float32, +) -> bool: + """Convert a grid column to a raster using gdal_rasterize. + + This function creates a raster from the study_area_grid layer, + burning values from the specified column into the output raster. + + Args: + gpkg_path: Path to the GeoPackage containing study_area_grid. + column_name: Name of the column to rasterize. + output_raster_path: Path for the output raster file. + cell_size: Cell size in map units (meters for projected CRS). + extent: Optional tuple of (xmin, ymin, xmax, ymax). If None, + computed from the grid layer extent. + nodata: NoData value for the output raster. Defaults to -9999. + area_name: Optional area name to filter grid cells. + output_type: GDAL data type for output. Defaults to GDT_Float32. + + Returns: + True if rasterization succeeded, False otherwise. + """ + if not os.path.exists(gpkg_path): + log_message(f"GeoPackage not found: {gpkg_path}", level=Qgis.Warning) + return False + + sanitized_column = _sanitize_column_name(column_name) + + # Build the layer specification with optional attribute filter + if area_name: + layer_spec = "study_area_grid" + where_clause = f"area_name = '{area_name}'" + else: + layer_spec = "study_area_grid" + where_clause = None + + try: + # Open the GeoPackage to get extent and CRS info + ds = ogr.Open(gpkg_path, 0) + if not ds: + log_message(f"Could not open GeoPackage: {gpkg_path}", level=Qgis.Critical) + return False + + layer = ds.GetLayerByName("study_area_grid") + if not layer: + log_message("study_area_grid layer not found", level=Qgis.Critical) + ds = None + return False + + # Apply filter to get correct extent + if area_name: + layer.SetAttributeFilter(f"area_name = '{area_name}'") + + # Get extent if not provided + if extent is None: + layer_extent = layer.GetExtent() + extent = (layer_extent[0], layer_extent[2], layer_extent[1], layer_extent[3]) + # extent is (xmin, ymin, xmax, ymax) + + # Get spatial reference + srs = layer.GetSpatialRef() + srs_wkt = srs.ExportToWkt() if srs else None + + # Reset filter + layer.SetAttributeFilter(None) + ds = None + + # Calculate raster dimensions + xmin, ymin, xmax, ymax = extent + width = int((xmax - xmin) / cell_size) + height = int((ymax - ymin) / cell_size) + + if width <= 0 or height <= 0: + log_message(f"Invalid raster dimensions: {width}x{height}", level=Qgis.Critical) + return False + + # Build gdal_rasterize options + rasterize_options = gdal.RasterizeOptions( + format="GTiff", + outputType=output_type, + width=width, + height=height, + outputBounds=[xmin, ymin, xmax, ymax], + noData=nodata, + initValues=[nodata], + attribute=sanitized_column, + layers=[layer_spec], + where=where_clause, + creationOptions=["COMPRESS=LZW", "TILED=YES"], + ) + + # Run rasterization + result = gdal.Rasterize( + output_raster_path, + gpkg_path, + options=rasterize_options, + ) + + if result is None: + log_message(f"gdal_rasterize failed for column {sanitized_column}", level=Qgis.Critical) + return False + + # Set spatial reference on output + if srs_wkt: + result.SetProjection(srs_wkt) + + # Ensure data is written + result.FlushCache() + result = None + + log_message(f"Rasterized column {sanitized_column} to {output_raster_path}") + return True + + except Exception as e: + log_message(f"Error in rasterize_grid_column: {e}", level=Qgis.Critical) + return False + + +def write_buffer_values_to_grid( + gpkg_path: str, + column_name: str, + buffer_layer: QgsVectorLayer, + value_field: str = "value", + aggregation_method: str = "MAX", + feedback: QgsFeedback = None, +) -> int: + """Write buffer polygon scores to intersecting grid cells. + + For each grid cell, finds intersecting buffer polygons and aggregates + their scores (using MAX by default) to determine the cell's value. + + Args: + gpkg_path: Path to the GeoPackage containing study_area_grid. + column_name: Name of the column to write values to. + buffer_layer: QgsVectorLayer containing buffer polygons with scores. + value_field: Name of the field containing scores in buffer_layer. + aggregation_method: How to combine multiple intersecting buffers + (MAX, MIN, AVG). Defaults to MAX. + feedback: Optional feedback for progress reporting. + + Returns: + Number of cells updated, or -1 on error. + """ + from qgis.core import QgsFeatureRequest, QgsSpatialIndex + + if not os.path.exists(gpkg_path): + log_message(f"GeoPackage not found: {gpkg_path}", level=Qgis.Warning) + return -1 + + if not buffer_layer or not buffer_layer.isValid(): + log_message("Invalid buffer layer provided", level=Qgis.Warning) + return -1 + + sanitized_column = _sanitize_column_name(column_name) + + try: + # Load grid layer + grid_layer = QgsVectorLayer(f"{gpkg_path}|layername=study_area_grid", "grid", "ogr") + if not grid_layer.isValid(): + log_message("Could not load study_area_grid layer", level=Qgis.Critical) + return -1 + + # Create spatial index for buffer layer + buffer_index = QgsSpatialIndex(buffer_layer.getFeatures()) + + # Collect scores per grid cell + grid_scores = {} + total_features = grid_layer.featureCount() + log_message(f"Processing {total_features} grid cells against buffer layer") + + for i, grid_feature in enumerate(grid_layer.getFeatures()): + grid_geom = grid_feature.geometry() + if grid_geom.isEmpty(): + continue + + grid_fid = grid_feature.id() + + # Find intersecting buffer polygons + candidate_ids = buffer_index.intersects(grid_geom.boundingBox()) + if not candidate_ids: + continue + + # Get actual intersecting features and their scores + scores = [] + request = QgsFeatureRequest().setFilterFids(candidate_ids) + for buffer_feature in buffer_layer.getFeatures(request): + buffer_geom = buffer_feature.geometry() + if buffer_geom.intersects(grid_geom): + score = buffer_feature.attribute(value_field) + if score is not None: + scores.append(float(score)) + + # Aggregate scores + if scores: + if aggregation_method == "MAX": + final_score = max(scores) + elif aggregation_method == "MIN": + final_score = min(scores) + elif aggregation_method == "AVG": + final_score = sum(scores) / len(scores) + else: + final_score = max(scores) + + grid_scores[grid_fid] = final_score + + if feedback and i % 1000 == 0: + feedback.setProgress((i / total_features) * 50) + + log_message(f"Found {len(grid_scores)} grid cells with intersecting buffers") + + # Update grid using SQL batched updates + ds = _open_gpkg_for_write(gpkg_path) + if not ds: + log_message(f"Could not open GeoPackage: {gpkg_path}", level=Qgis.Critical) + return -1 + + updated_count = 0 + batch_size = 500 + fids = list(grid_scores.keys()) + + for batch_start in range(0, len(fids), batch_size): + batch_fids = fids[batch_start : batch_start + batch_size] + + # Build CASE statement for this batch + case_parts = [] + for fid in batch_fids: + score = grid_scores[fid] + case_parts.append(f"WHEN fid = {fid} THEN {score}") + + if case_parts: + fid_list = ",".join(str(f) for f in batch_fids) + sql = ( + f"UPDATE study_area_grid " # nosec B608 + f'SET "{sanitized_column}" = CASE {" ".join(case_parts)} END ' + f"WHERE fid IN ({fid_list})" + ) + _execute_sql_with_retry(ds, sql) + updated_count += len(batch_fids) + + if feedback: + progress = 50 + (batch_start / max(len(fids), 1)) * 50 + feedback.setProgress(progress) + + ds = None + log_message(f"Updated {updated_count} grid cells with buffer scores") + return updated_count + + except Exception as e: + log_message(f"Error in write_buffer_values_to_grid: {e}", level=Qgis.Critical) + return -1 + + +def get_grid_column_statistics( + gpkg_path: str, + column_name: str, + area_name: Optional[str] = None, +) -> Dict[str, Any]: + """Calculate statistics for a grid column. + + Args: + gpkg_path: Path to the GeoPackage containing study_area_grid. + column_name: Name of the column to analyze. + area_name: Optional area name to filter grid cells. + + Returns: + Dict with keys: count, min, max, mean, sum, null_count. + Returns empty dict on error. + """ + if not os.path.exists(gpkg_path): + log_message(f"GeoPackage not found: {gpkg_path}", level=Qgis.Warning) + return {} + + try: + ds = ogr.Open(gpkg_path, 0) + if not ds: + log_message(f"Could not open GeoPackage: {gpkg_path}", level=Qgis.Critical) + return {} + + layer, field_idx = _get_grid_layer_and_field_index(ds, column_name) + if layer is None or field_idx < 0: + ds = None + return {} + + sanitized_column = _sanitize_column_name(column_name) + + # Set attribute filter + if area_name: + layer.SetAttributeFilter(f"area_name = '{area_name}'") + + # Calculate statistics + values = [] + null_count = 0 + + for feature in layer: + value = feature.GetField(sanitized_column) + if value is None: + null_count += 1 + else: + values.append(float(value)) + + layer.SetAttributeFilter(None) + ds = None + + if not values: + return { + "count": 0, + "min": None, + "max": None, + "mean": None, + "sum": None, + "null_count": null_count, + } + + return { + "count": len(values), + "min": min(values), + "max": max(values), + "mean": sum(values) / len(values), + "sum": sum(values), + "null_count": null_count, + } + + except Exception as e: + log_message(f"Error in get_grid_column_statistics: {e}", level=Qgis.Critical) + return {} diff --git a/geest/core/h3_utils.py b/geest/core/h3_utils.py index 9f89b07c..baa8bdc4 100644 --- a/geest/core/h3_utils.py +++ b/geest/core/h3_utils.py @@ -16,6 +16,27 @@ from geest.utilities import log_message +# Fallback average H3 cell areas in km^2 (approximate) +H3_CELL_AREA_KM2 = { + 0: 4250546.848, + 1: 607220.978, + 2: 86745.854, + 3: 12392.265, + 4: 1770.323, + 5: 252.903, + 6: 36.129, + 7: 5.161, + 8: 0.737, + 9: 0.105, + 10: 0.015, + 11: 0.002, + 12: 0.00031, + 13: 0.000044, + 14: 0.000006, + 15: 0.0000009, +} + + def get_h3_resolution_for_scale(analysis_scale: str) -> Optional[int]: """Get H3 resolution for a given analysis scale. @@ -30,6 +51,80 @@ def get_h3_resolution_for_scale(analysis_scale: str) -> Optional[int]: return None +def h3_cell_area_km2(h3_resolution: int) -> float: + """Return average H3 cell area in km^2 for a resolution. + + Args: + h3_resolution: H3 resolution level. + + Returns: + Average hexagon area in square kilometers. + """ + if h3_resolution < 0 or h3_resolution > 15: + raise ValueError("H3 resolution must be between 0 and 15.") + + try: + import h3 + + return float(h3.average_hexagon_area(h3_resolution, unit="km^2")) + except Exception: + return H3_CELL_AREA_KM2[h3_resolution] + + +def estimate_h3_cells_for_area(area_km2: float, h3_resolution: int) -> int: + """Estimate H3 cell count needed for an area. + + Args: + area_km2: Area in square kilometers. + h3_resolution: H3 resolution level. + + Returns: + Estimated number of cells. + """ + if area_km2 <= 0: + return 0 + + cell_area = h3_cell_area_km2(h3_resolution) + if cell_area <= 0: + return 0 + + return max(1, int(round(area_km2 / cell_area))) + + +def suggest_coarser_resolution(area_km2: float, max_cells: int, start_res: int) -> int: + """Suggest a coarser resolution that stays under max cell count. + + Args: + area_km2: Area in square kilometers. + max_cells: Maximum allowed estimated cells. + start_res: Starting resolution. + + Returns: + Suggested resolution (0-15). + """ + suggested = max(0, min(15, start_res)) + while suggested > 0 and estimate_h3_cells_for_area(area_km2, suggested) > max_cells: + suggested -= 1 + return suggested + + +def suggest_finer_resolution(area_km2: float, min_cells: int, start_res: int) -> int: + """Suggest a finer resolution that reaches minimum cell count. + + Args: + area_km2: Area in square kilometers. + min_cells: Minimum estimated cells. + start_res: Starting resolution. + + Returns: + Suggested resolution (0-15). + """ + suggested = max(0, min(15, start_res)) + while suggested < 15 and estimate_h3_cells_for_area(area_km2, suggested) < min_cells: + suggested += 1 + return suggested + + def bbox_to_wgs84( xmin: float, xmax: float, diff --git a/geest/core/json_tree_item.py b/geest/core/json_tree_item.py index b8660c1e..e6053c77 100644 --- a/geest/core/json_tree_item.py +++ b/geest/core/json_tree_item.py @@ -10,7 +10,7 @@ from typing import Optional from qgis.core import Qgis -from qgis.PyQt.QtCore import Qt, QReadWriteLock, QReadLocker, QWriteLocker +from qgis.PyQt.QtCore import QReadWriteLock, Qt from qgis.PyQt.QtGui import QColor, QFont, QIcon from geest.core.settings import setting @@ -142,10 +142,61 @@ def is_enabled(self) -> bool: return self._enabled def is_only_child(self) -> bool: - """Returns the only child status of this item.""" - siblings_count = len(self.parentItem.childItems) - if siblings_count == 1: - return True + """Returns True if this item is the only child of its parent.""" + if not self.parentItem: + return False + return len(self.parentItem.childItems) == 1 + + def visible_row(self) -> int: + """Returns the visible row position among siblings (excluding hidden siblings). + + This is used when hidden items need to be skipped in the visual tree. + """ + if not self.parentItem: + return 0 + visible_siblings = [c for c in self.parentItem.childItems if c.is_visible()] + if self in visible_siblings: + return visible_siblings.index(self) + return 0 + + def get_effective_visible_children(self) -> list: + """Returns effective visible children for tree display. + + For normal visible children, returns them directly. + For hidden children that have exactly one child, returns their child instead + (promoting the grandchild to appear at this level). + This enables hiding single-child factors while showing their indicator. + + Returns: + list: List of JsonTreeItem objects to display as children. + """ + effective_children = [] + for child in self.childItems: + if child.is_visible(): + effective_children.append(child) + elif len(child.childItems) == 1: + # Hidden single-child item: promote its child to this level + grandchild = child.childItems[0] + if grandchild.is_visible(): + effective_children.append(grandchild) + return effective_children + + def get_visual_parent(self): + """Returns the visual parent for tree display. + + If the actual parent is hidden, returns the grandparent instead. + This handles the case where factors are hidden but indicators need + to appear under the dimension. + + Returns: + JsonTreeItem: The visual parent item. + """ + if not self.parentItem: + return None + if self.parentItem.is_visible(): + return self.parentItem + # Parent is hidden, return grandparent + return self.parentItem.parentItem def internalPointer(self): """Returns a reference to itself, or any unique identifier for the item.""" diff --git a/geest/core/reports/base_report.py b/geest/core/reports/base_report.py index 5f7730bb..eb027600 100644 --- a/geest/core/reports/base_report.py +++ b/geest/core/reports/base_report.py @@ -499,10 +499,13 @@ def add_header_and_footer(self, page_number, title: str = ""): footer_text = """

This plugin was built with support from the Canada Clean Energy and - Forest Climate Facility (CCEFCF) and the Global Development Fund (GDF), + Forest Climate Facility (CCEFCF) and the Global Data Facility (GDF), by the Geospatial Team in the Development Economics Data Group (DECDG). This project is open source; you can download the code at https://github.com/worldbank/GEOE3.

+

Data attribution: analysis workflows may include data/services from OpenStreetMap, + OpenRouteService, GHSL, Ookla Open Data, Space2Stats, ACLED, VIIRS Nighttime Lights, + and user-supplied datasets. Please review source terms and citation requirements.

""" credits_text = """Developed by Kartoza for and with The World Bank.""" diff --git a/geest/core/s2s_client.py b/geest/core/s2s_client.py new file mode 100644 index 00000000..83c68963 --- /dev/null +++ b/geest/core/s2s_client.py @@ -0,0 +1,292 @@ +# -*- coding: utf-8 -*- +"""Client for querying the public Space2Stats API.""" + +import json +import random +import time +from typing import Any, Dict, List, Optional + +from qgis.core import QgsNetworkAccessManager +from qgis.PyQt.QtCore import QObject, QUrl +from qgis.PyQt.QtNetwork import QNetworkRequest + + +class S2SClient(QObject): + """Client wrapper for Space2Stats endpoints used by GeoE3. + + This client provides thin request/response helpers for the public API. + It intentionally keeps scope small for phase 1 integration. + """ + + VALID_JOIN_METHODS = {"touches", "centroid", "within"} + VALID_GEOMETRIES = {"point", "polygon"} + RETRYABLE_STATUS_CODES = {502, 503, 504} + + def __init__( + self, + base_url: Optional[str] = None, + max_attempts: int = 4, + backoff_base_seconds: float = 0.5, + backoff_jitter_seconds: float = 0.2, + ): + """Initialize the S2S client. + + Args: + base_url: API base URL. Defaults to public Space2Stats host. + max_attempts: Maximum number of attempts for transient failures. + backoff_base_seconds: Base exponential backoff delay. + backoff_jitter_seconds: Random jitter added to each backoff delay. + """ + super().__init__() + self.base_url = (base_url or "https://space2stats.ds.io").rstrip("/") + self.network_manager = QgsNetworkAccessManager.instance() + self.max_attempts = max(1, int(max_attempts)) + self.backoff_base_seconds = max(0.0, float(backoff_base_seconds)) + self.backoff_jitter_seconds = max(0.0, float(backoff_jitter_seconds)) + + def health(self) -> Dict[str, Any]: + """Check API health endpoint. + + Returns: + Parsed JSON object from the health endpoint. + """ + result = self._request("GET", "/health") + if not isinstance(result, dict): + raise RuntimeError("Unexpected /health response format.") + return result + + def fields(self) -> List[str]: + """Fetch available summary fields from S2S. + + Returns: + List of field names. + """ + result = self._request("GET", "/fields") + if isinstance(result, list): + return [str(value) for value in result] + + if isinstance(result, dict): + for key in ("fields", "data"): + value = result.get(key) + if isinstance(value, list): + return [str(item) for item in value] + keys = ", ".join(sorted(result.keys())[:8]) + raise RuntimeError(f"Unexpected /fields response format: object keys [{keys}]") + + raise RuntimeError(f"Unexpected /fields response format: {type(result).__name__}") + + def summary( + self, + aoi: Dict[str, Any], + fields: List[str], + spatial_join_method: str = "centroid", + geometry: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """Query per-hex summary records for an AOI. + + Args: + aoi: GeoJSON feature (Polygon/MultiPolygon). + fields: S2S field names to fetch. + spatial_join_method: One of touches/centroid/within. + geometry: Optional geometry type (point or polygon). + + Returns: + List of summary rows, typically including ``hex_id`` and selected fields. + """ + if not isinstance(aoi, dict) or not aoi: + raise ValueError("'aoi' must be a non-empty GeoJSON feature dictionary.") + + if not isinstance(fields, list) or not fields: + raise ValueError("'fields' must be a non-empty list of field names.") + + if spatial_join_method not in self.VALID_JOIN_METHODS: + raise ValueError( + f"Invalid spatial_join_method '{spatial_join_method}'. " + f"Use one of: {sorted(self.VALID_JOIN_METHODS)}" + ) + + if geometry is not None and geometry not in self.VALID_GEOMETRIES: + raise ValueError(f"Invalid geometry '{geometry}'. Use one of: {sorted(self.VALID_GEOMETRIES)}") + + payload: Dict[str, Any] = { + "aoi": aoi, + "spatial_join_method": spatial_join_method, + "fields": fields, + } + if geometry is not None: + payload["geometry"] = geometry + + result = self._request("POST", "/summary", payload) + if not isinstance(result, list): + raise RuntimeError("Unexpected /summary response format.") + return result + + def summary_by_hexids( + self, + hex_ids: List[str], + fields: List[str], + geometry: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """Query per-hex summary records for explicit H3 hex IDs. + + Args: + hex_ids: List of H3 hex IDs. + fields: S2S field names to fetch. + geometry: Optional geometry type (point or polygon). + + Returns: + List of summary rows, typically including ``hex_id`` and selected fields. + """ + if not isinstance(hex_ids, list) or not hex_ids: + raise ValueError("'hex_ids' must be a non-empty list of H3 IDs.") + + normalized_hex_ids = [str(value).strip() for value in hex_ids if str(value).strip()] + if not normalized_hex_ids: + raise ValueError("'hex_ids' must contain at least one non-empty H3 ID.") + + if not isinstance(fields, list) or not fields: + raise ValueError("'fields' must be a non-empty list of field names.") + + if geometry is not None and geometry not in self.VALID_GEOMETRIES: + raise ValueError(f"Invalid geometry '{geometry}'. Use one of: {sorted(self.VALID_GEOMETRIES)}") + + payload: Dict[str, Any] = { + "hex_ids": normalized_hex_ids, + "fields": fields, + } + if geometry is not None: + payload["geometry"] = geometry + + result = self._request("POST", "/summary_by_hexids", payload) + if not isinstance(result, list): + raise RuntimeError("Unexpected /summary_by_hexids response format.") + return result + + def _request(self, method: str, endpoint: str, payload: Optional[Dict[str, Any]] = None) -> Any: + """Execute a blocking JSON request to S2S. + + Args: + method: HTTP method (GET or POST). + endpoint: API endpoint path. + payload: Optional JSON payload for POST requests. + + Returns: + Parsed JSON response payload. + """ + if method not in {"GET", "POST"}: + raise ValueError(f"Unsupported method: {method}") + + last_error: Optional[Exception] = None + + for attempt in range(1, self.max_attempts + 1): + url = QUrl(f"{self.base_url}{endpoint}") + request = QNetworkRequest(url) + request.setHeader(QNetworkRequest.ContentTypeHeader, "application/json") + + try: + if method == "GET": + reply = self.network_manager.blockingGet(request) + else: + data = json.dumps(payload or {}).encode("utf-8") + reply = self.network_manager.blockingPost(request, data) + + status_code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) + response_text = self._extract_text(reply.content()) + + if status_code is None: + raise RuntimeError("No HTTP status code received from S2S API.") + + if status_code == 422: + raise ValueError(f"S2S request validation failed (422): {response_text}") + if status_code == 429: + raise RuntimeError("S2S API rate limit exceeded (429). Please retry later.") + if status_code >= 500: + if status_code in self.RETRYABLE_STATUS_CODES and attempt < self.max_attempts: + self._sleep_before_retry(attempt) + continue + + if status_code in self.RETRYABLE_STATUS_CODES: + raise RuntimeError( + f"S2S service temporarily unavailable ({status_code}) after {attempt} attempts." + ) + + raise RuntimeError(f"S2S server error ({status_code}).") + if status_code >= 400: + raise RuntimeError(f"S2S request failed ({status_code}): {response_text}") + + try: + return self._parse_json_response(response_text) + except json.JSONDecodeError as error: + raise RuntimeError(f"Failed to parse S2S JSON response: {error}") from error + + except Exception as error: + if not self._is_retryable_error(error): + raise + + last_error = error + if attempt >= self.max_attempts: + raise RuntimeError( + f"S2S service temporarily unavailable after {attempt} attempts: {error}" + ) from error + + self._sleep_before_retry(attempt) + + raise RuntimeError(f"S2S request failed after retries: {last_error}") + + def _sleep_before_retry(self, attempt: int) -> None: + """Sleep with exponential backoff before retrying.""" + delay = self.backoff_base_seconds * (2 ** max(0, attempt - 1)) + if self.backoff_jitter_seconds > 0: + delay += random.uniform(0.0, self.backoff_jitter_seconds) + if delay > 0: + time.sleep(delay) + + @staticmethod + def _is_retryable_error(error: Exception) -> bool: + """Return True when an error is considered transient and retryable.""" + message = str(error).lower() + return ( + "no http status code received" in message + or "service temporarily unavailable" in message + or "connection" in message + or "timed out" in message + or "timeout" in message + or "failed to parse s2s json response" in message + or "extra data" in message + ) + + @staticmethod + def _parse_json_response(response_text: str) -> Any: + """Parse JSON responses, tolerating trailing junk after valid JSON. + + Some upstream responses intermittently append extra bytes after valid JSON. + We parse the first valid JSON document to keep requests resilient. + """ + normalized = response_text.lstrip("\ufeff\x00 \t\r\n") + if not normalized: + raise json.JSONDecodeError("Expecting value", response_text, 0) + + try: + return json.loads(normalized) + except json.JSONDecodeError: + decoder = json.JSONDecoder() + value, end = decoder.raw_decode(normalized) + remainder = normalized[end:].lstrip("\x00 \t\r\n") + if not remainder: + return value + + if remainder[0] in '{["-0123456789tfn': + raise json.JSONDecodeError("Extra data", normalized, end) + + return value + + @staticmethod + def _extract_text(content: Any) -> str: + """Convert network reply content to UTF-8 text.""" + if isinstance(content, (bytes, bytearray)): + return bytes(content).decode("utf-8") + + if hasattr(content, "data"): + return bytes(content).decode("utf-8") + + return str(content) diff --git a/geest/core/s2s_task_gate.py b/geest/core/s2s_task_gate.py new file mode 100644 index 00000000..067f2260 --- /dev/null +++ b/geest/core/s2s_task_gate.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +"""Global in-process gate for S2S downloader tasks.""" + +import time +import uuid +from typing import Optional + + +__copyright__ = "Copyright 2024, Tim Sutton" +__license__ = "GPL version 3" +__email__ = "tim@kartoza.com" +__revision__ = "$Format:%H$" + + +class S2STaskGate: + """Simple global mutex-like gate for S2S task execution.""" + + MAX_STALE_SECONDS = 60 * 60 * 2 + + _active_token: Optional[str] = None + _active_label: str = "" + _active_started_at: float = 0.0 + + @classmethod + def acquire(cls, label: str) -> Optional[str]: + """Acquire the global gate and return a token, or None if busy.""" + cls._clear_stale_lock() + if cls._active_token: + return None + + cls._active_token = uuid.uuid4().hex + cls._active_label = str(label or "").strip() + cls._active_started_at = time.monotonic() + return cls._active_token + + @classmethod + def release(cls, token: Optional[str]) -> None: + """Release the gate if token matches the current active token.""" + if token and token == cls._active_token: + cls._active_token = None + cls._active_label = "" + cls._active_started_at = 0.0 + + @classmethod + def active_label(cls) -> str: + """Return human-readable label for the active owner.""" + return cls._active_label + + @classmethod + def _clear_stale_lock(cls) -> None: + """Clear stale lock state if it has been held unusually long.""" + if not cls._active_token or cls._active_started_at <= 0: + return + + elapsed = time.monotonic() - cls._active_started_at + if elapsed > cls.MAX_STALE_SECONDS: + cls._active_token = None + cls._active_label = "" + cls._active_started_at = 0.0 diff --git a/geest/core/tasks/__init__.py b/geest/core/tasks/__init__.py index a2a152fd..b9f4fffd 100644 --- a/geest/core/tasks/__init__.py +++ b/geest/core/tasks/__init__.py @@ -15,3 +15,4 @@ from .study_area_report_task import StudyAreaReportTask from .analysis_report_task import AnalysisReportTask from .ghsl_downloader_task import GHSLDownloaderTask +from .s2s_downloader_task import S2SDownloaderTask diff --git a/geest/core/tasks/ghsl_downloader_task.py b/geest/core/tasks/ghsl_downloader_task.py index 707b8be4..7d55c57b 100644 --- a/geest/core/tasks/ghsl_downloader_task.py +++ b/geest/core/tasks/ghsl_downloader_task.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""📦 Ghsl Downloader Task module. +"""📦 GHSL Downloader Task module. This module contains functionality for ghsl downloader task. """ diff --git a/geest/core/tasks/grid_from_bbox_h3_task.py b/geest/core/tasks/grid_from_bbox_h3_task.py index b16bca3d..13f266e1 100644 --- a/geest/core/tasks/grid_from_bbox_h3_task.py +++ b/geest/core/tasks/grid_from_bbox_h3_task.py @@ -157,10 +157,8 @@ def run(self) -> bool: # Check precise intersection with study area geometry if self.geom.Intersects(polygon): - # Clip hexagon to study area boundary for exact alignment - clipped_polygon = self.geom.Intersection(polygon) - if clipped_polygon and not clipped_polygon.IsEmpty(): - self.features_out.append((h3_index, clipped_polygon)) + # Keep full H3 hexagon geometry (do not clip to study area boundary) + self.features_out.append((h3_index, polygon)) processed += 1 diff --git a/geest/core/tasks/s2s_downloader_task.py b/geest/core/tasks/s2s_downloader_task.py new file mode 100644 index 00000000..7519d4a5 --- /dev/null +++ b/geest/core/tasks/s2s_downloader_task.py @@ -0,0 +1,433 @@ +# -*- coding: utf-8 -*- +"""Space2Stats downloader task.""" + +import datetime +import json +import os +import traceback +import uuid +from typing import Any, Dict, List, Optional, Tuple + +from osgeo import ogr, osr +from qgis.core import QgsFeedback, QgsTask +from qgis.PyQt.QtCore import pyqtSignal + +from geest.core.s2s_client import S2SClient +from geest.utilities import log_message + + +class S2SDownloaderTask(QgsTask): + """A QgsTask for downloading and persisting Space2Stats summary data.""" + + error_occurred = pyqtSignal(str) + progress_updated = pyqtSignal(str) + chunk_completed = pyqtSignal(int, int) + + def __init__( + self, + aoi: Dict[str, Any], + fields: List[str], + working_dir: str, + filename: str = "s2s_summary", + spatial_join_method: str = "centroid", + geometry: Optional[str] = "point", + base_url: Optional[str] = None, + delete_existing: bool = True, + feedback: Optional[QgsFeedback] = None, + mode: str = "aoi", + hex_ids: Optional[List[str]] = None, + chunk_size: int = 3000, + start_chunk_index: int = 0, + append_existing: bool = False, + ): + """Initialize S2S downloader task. + + Args: + aoi: GeoJSON feature polygon/multipolygon. + fields: S2S fields to fetch. + working_dir: Project working directory. + filename: Output file basename (without extension). + spatial_join_method: touches, centroid, or within. + geometry: Optional geometry mode (point/polygon) for S2S response. + base_url: Optional S2S API base URL override. + delete_existing: Remove existing output file before writing. + feedback: Optional feedback object. + mode: Fetch mode ('aoi' or 'hex_ids'). + hex_ids: Optional H3 IDs for chunked mode. + chunk_size: Max H3 IDs per chunk in chunked mode. + start_chunk_index: Chunk index to resume from in chunked mode. + append_existing: Append into existing output when resuming chunked mode. + """ + super().__init__("S2S Downloader Task", QgsTask.CanCancel) + + if not working_dir: + raise ValueError("Working directory cannot be empty") + if not isinstance(fields, list) or not fields: + raise ValueError("Fields must be a non-empty list") + if mode == "aoi" and (not isinstance(aoi, dict) or not aoi): + raise ValueError("AOI must be a non-empty GeoJSON feature") + if mode not in {"aoi", "hex_ids"}: + raise ValueError("Mode must be 'aoi' or 'hex_ids'") + if mode == "hex_ids" and (not isinstance(hex_ids, list) or not hex_ids): + raise ValueError("hex_ids must be a non-empty list when mode='hex_ids'") + + self.aoi = aoi + self.fields = fields + self.working_dir = working_dir + self.filename = filename + self.spatial_join_method = spatial_join_method + self.geometry = geometry + self.base_url = base_url + self.delete_existing = delete_existing + self.feedback = feedback if feedback else QgsFeedback() + self.mode = mode + self.hex_ids = [str(hex_id).strip() for hex_id in (hex_ids or []) if str(hex_id).strip()] + self.chunk_size = max(1, int(chunk_size)) + self.start_chunk_index = max(0, int(start_chunk_index)) + self.append_existing = append_existing + + self.study_area_dir = os.path.join(self.working_dir, "study_area") + self.output_path = os.path.join(self.study_area_dir, f"{self.filename}.gpkg") + self.layer_name = self.filename + self._temp_output_path = "" + + self._create_output_directory() + + def run(self) -> bool: + """Execute task in worker thread.""" + try: + self.setProgress(1) + self.progress_updated.emit("Checking S2S API health...") + client = S2SClient(base_url=self.base_url) + client.health() + + if self.isCanceled(): + return False + + self.setProgress(10) + self.progress_updated.emit("Validating requested S2S fields...") + try: + available_fields = set(client.fields()) + missing_fields = [field for field in self.fields if field not in available_fields] + if missing_fields: + raise ValueError(f"Requested S2S fields are unavailable: {', '.join(missing_fields)}") + except ValueError: + raise + except Exception as fields_error: + self.progress_updated.emit("S2S field metadata unavailable, continuing with requested fields...") + log_message( + f"S2S fields validation skipped due to transient metadata error: {fields_error}", + level="WARNING", + ) + + if self.isCanceled(): + return False + + self.setProgress(25) + if self.mode == "hex_ids": + self.progress_updated.emit("Fetching S2S summary data by H3 chunks...") + self._run_hex_ids_mode(client) + else: + self.progress_updated.emit("Fetching S2S summary data...") + rows = client.summary( + aoi=self.aoi, + fields=self.fields, + spatial_join_method=self.spatial_join_method, + geometry=self.geometry, + ) + + if not rows: + raise ValueError("S2S returned no rows for the provided AOI and fields.") + + if self.isCanceled(): + return False + + self.setProgress(70) + self.progress_updated.emit("Writing S2S output to GeoPackage...") + self._write_rows_to_gpkg(rows) + + self.setProgress(100) + self.progress_updated.emit("S2S download complete.") + log_message(f"S2S output written to {self.output_path}") + return True + + except Exception as error: + message = f"Error in S2SDownloaderTask: {error}" + log_message(message) + log_message(traceback.format_exc()) + self.error_occurred.emit(message) + self._cleanup_partial_output() + self._write_error_file(traceback.format_exc()) + return False + + def _run_hex_ids_mode(self, client: S2SClient) -> None: + """Fetch S2S summary in chunks using explicit H3 IDs.""" + chunks = self._chunk_hex_ids(self.hex_ids, self.chunk_size) + total_chunks = len(chunks) + if total_chunks == 0: + raise ValueError("No hex IDs available for chunked S2S fetch.") + + if self.start_chunk_index >= total_chunks: + raise ValueError(f"Start chunk index {self.start_chunk_index} exceeds total chunks {total_chunks}.") + + if ( + self.delete_existing + and not self.append_existing + and self.start_chunk_index == 0 + and os.path.exists(self.output_path) + ): + os.remove(self.output_path) + + wrote_any_rows = False + for chunk_index in range(self.start_chunk_index, total_chunks): + if self.isCanceled(): + return + + current_chunk = chunk_index + 1 + self.progress_updated.emit(f"Fetching S2S chunk {current_chunk}/{total_chunks}...") + rows = client.summary_by_hexids( + hex_ids=chunks[chunk_index], + fields=self.fields, + geometry=self.geometry, + ) + + if rows: + self.progress_updated.emit(f"Writing S2S chunk {current_chunk}/{total_chunks}...") + if os.path.exists(self.output_path): + self._append_rows_to_gpkg(rows) + else: + self._write_rows_to_gpkg(rows) + wrote_any_rows = True + + self.chunk_completed.emit(current_chunk, total_chunks) + progress = 25 + int((current_chunk / total_chunks) * 75) + self.setProgress(progress) + + if not wrote_any_rows: + raise ValueError("S2S returned no rows for the provided hex IDs and fields.") + + @staticmethod + def _chunk_hex_ids(hex_ids: List[str], chunk_size: int) -> List[List[str]]: + """Split H3 IDs into fixed-size chunks.""" + return [hex_ids[i : i + chunk_size] for i in range(0, len(hex_ids), chunk_size)] + + def _create_output_directory(self) -> None: + """Create study area directory if needed.""" + os.makedirs(self.study_area_dir, exist_ok=True) + + def _cleanup_partial_output(self) -> None: + """Remove partial output file on failure.""" + if self._temp_output_path and os.path.exists(self._temp_output_path): + try: + os.remove(self._temp_output_path) + except Exception as cleanup_error: + log_message(f"Could not remove temporary S2S output: {cleanup_error}") + self._temp_output_path = "" + + def _write_error_file(self, stack_trace: str) -> None: + """Write a task error trace in the working directory.""" + try: + error_file = os.path.join(self.working_dir, "s2s_download_error.txt") + with open(error_file, "w", encoding="utf-8") as handle: + handle.write(f"{datetime.datetime.now()}\n") + handle.write(f"Output Path: {self.output_path}\n") + handle.write(stack_trace) + except Exception: + pass + + def _write_rows_to_gpkg(self, rows: List[Dict[str, Any]]) -> None: + """Persist S2S rows to a GeoPackage layer.""" + driver = ogr.GetDriverByName("GPKG") + if driver is None: + raise RuntimeError("GeoPackage driver is not available.") + + if os.path.exists(self.output_path) and not self.delete_existing: + raise RuntimeError(f"Output already exists and delete_existing is False: {self.output_path}") + + temp_filename = f"{self.filename}.{uuid.uuid4().hex}.tmp.gpkg" + self._temp_output_path = os.path.join(self.study_area_dir, temp_filename) + + dataset = driver.CreateDataSource(self._temp_output_path) + if dataset is None: + raise RuntimeError(f"Could not create output GeoPackage: {self._temp_output_path}") + + try: + geometry_type = self._infer_geometry_type(rows) + spatial_ref = None + if geometry_type != ogr.wkbNone: + spatial_ref = osr.SpatialReference() + spatial_ref.ImportFromEPSG(4326) + + layer = dataset.CreateLayer(self.layer_name, srs=spatial_ref, geom_type=geometry_type) + if layer is None: + raise RuntimeError("Could not create S2S output layer.") + + output_fields = ["hex_id"] + [field for field in self.fields if field != "hex_id"] + output_field_types = self._infer_field_types(rows, output_fields) + + for field_name in output_fields: + field_defn = ogr.FieldDefn(field_name, output_field_types[field_name]) + if output_field_types[field_name] == ogr.OFTReal: + field_defn.SetWidth(32) + field_defn.SetPrecision(12) + layer.CreateField(field_defn) + + layer_defn = layer.GetLayerDefn() + total = len(rows) + for index, row in enumerate(rows): + if self.isCanceled(): + raise RuntimeError("S2S task was cancelled during output write.") + + feature = ogr.Feature(layer_defn) + for field_name in output_fields: + value = row.get(field_name) + if value is None: + continue + if isinstance(value, bool): + feature.SetField(field_name, int(value)) + elif isinstance(value, (int, float, str)): + feature.SetField(field_name, value) + else: + feature.SetField(field_name, json.dumps(value)) + + geometry_value = row.get("geometry") + if geometry_value is not None and geometry_type != ogr.wkbNone: + normalized_geometry = self._normalize_geometry(geometry_value) + geometry = None + if normalized_geometry is not None: + geometry = ogr.CreateGeometryFromJson(json.dumps(normalized_geometry)) + if geometry is not None: + feature.SetGeometry(geometry) + + if layer.CreateFeature(feature) != 0: + raise RuntimeError("Failed to create feature in S2S output layer.") + + progress = 70 + int(((index + 1) / total) * 30) + self.setProgress(progress) + feature = None + + finally: + dataset = None + + os.replace(self._temp_output_path, self.output_path) + self._temp_output_path = "" + + def _append_rows_to_gpkg(self, rows: List[Dict[str, Any]]) -> None: + """Append S2S rows to an existing GeoPackage layer.""" + if not os.path.exists(self.output_path): + self._write_rows_to_gpkg(rows) + return + + dataset = ogr.Open(self.output_path, 1) + if dataset is None: + raise RuntimeError(f"Could not open output GeoPackage for append: {self.output_path}") + + try: + layer = dataset.GetLayerByName(self.layer_name) + if layer is None: + layer = dataset.GetLayer(0) + if layer is None: + raise RuntimeError("Could not open target layer for S2S append.") + + layer_defn = layer.GetLayerDefn() + known_fields = [layer_defn.GetFieldDefn(i).GetNameRef() for i in range(layer_defn.GetFieldCount())] + total = len(rows) + + for index, row in enumerate(rows): + if self.isCanceled(): + raise RuntimeError("S2S task was cancelled during append write.") + + feature = ogr.Feature(layer_defn) + for field_name in known_fields: + if field_name == "geometry": + continue + value = row.get(field_name) + if value is None: + continue + if isinstance(value, bool): + feature.SetField(field_name, int(value)) + elif isinstance(value, (int, float, str)): + feature.SetField(field_name, value) + else: + feature.SetField(field_name, json.dumps(value)) + + geometry_value = row.get("geometry") + if geometry_value is not None: + normalized_geometry = self._normalize_geometry(geometry_value) + geometry = None + if normalized_geometry is not None: + geometry = ogr.CreateGeometryFromJson(json.dumps(normalized_geometry)) + if geometry is not None: + feature.SetGeometry(geometry) + + if layer.CreateFeature(feature) != 0: + raise RuntimeError("Failed to append feature in S2S output layer.") + + chunk_progress = int(((index + 1) / total) * 100) + self.setProgress(min(99, max(25, chunk_progress))) + feature = None + finally: + dataset = None + + @staticmethod + def _infer_geometry_type(rows: List[Dict[str, Any]]) -> int: + """Infer OGR geometry type from S2S rows.""" + for row in rows: + geometry = S2SDownloaderTask._normalize_geometry(row.get("geometry")) + if not geometry: + continue + + geom_type = str(geometry.get("type", "")).lower() + if geom_type == "point": + return ogr.wkbPoint + if geom_type == "polygon": + return ogr.wkbPolygon + if geom_type == "multipolygon": + return ogr.wkbMultiPolygon + return ogr.wkbUnknown + + return ogr.wkbNone + + @staticmethod + def _normalize_geometry(geometry_value: Any) -> Optional[Dict[str, Any]]: + """Normalize geometry values from S2S rows to GeoJSON dicts.""" + if geometry_value is None: + return None + + if isinstance(geometry_value, dict): + return geometry_value + + if isinstance(geometry_value, str): + try: + parsed = json.loads(geometry_value) + if isinstance(parsed, dict): + return parsed + except json.JSONDecodeError: + return None + + return None + + @staticmethod + def _infer_field_types(rows: List[Dict[str, Any]], output_fields: List[str]) -> Dict[str, int]: + """Infer OGR field types from returned rows.""" + inferred = {field: ogr.OFTString for field in output_fields} + + for field_name in output_fields: + for row in rows: + value = row.get(field_name) + if value is None: + continue + + if field_name == "hex_id": + inferred[field_name] = ogr.OFTString + elif isinstance(value, bool): + inferred[field_name] = ogr.OFTInteger + elif isinstance(value, int): + inferred[field_name] = ogr.OFTInteger64 + elif isinstance(value, float): + inferred[field_name] = ogr.OFTReal + else: + inferred[field_name] = ogr.OFTString + break + + return inferred diff --git a/geest/core/tasks/study_area_processing_task.py b/geest/core/tasks/study_area_processing_task.py index 1d82e959..7de0fca9 100644 --- a/geest/core/tasks/study_area_processing_task.py +++ b/geest/core/tasks/study_area_processing_task.py @@ -31,13 +31,14 @@ ) from geest.core.algorithms import GHSLDownloader, GHSLProcessor +from geest.core.grid_column_utils import add_model_columns_to_grid +from geest.core.h3_utils import estimate_h3_cells_for_area, get_h3_resolution_for_scale, h3_cell_area_km2 from geest.core.settings import setting -from geest.core.h3_utils import get_h3_resolution_for_scale from geest.utilities import calculate_utm_zone, log_message from .grid_chunker_task import GridChunkerTask -from .grid_from_bbox_task import GridFromBboxTask from .grid_from_bbox_h3_task import GridFromBboxH3Task +from .grid_from_bbox_task import GridFromBboxTask class QtQueue: @@ -390,6 +391,7 @@ def run(self): if self.ds: try: self.ds.FlushCache() + self.ds.ExecuteSQL("PRAGMA wal_checkpoint(TRUNCATE)") except Exception as e: log_message(f"UnifiedWriter: Error flushing on cleanup: {e}", level="WARNING") self.ds = None @@ -709,7 +711,15 @@ class ChunkRunnable(QRunnable): """ def __init__( - self, chunk, geom, cell_size, feedback, write_callback=None, analysis_scale="national", epsg_code=None + self, + chunk, + geom, + cell_size, + feedback, + write_callback=None, + analysis_scale="national", + epsg_code=None, + h3_resolution=6, ): """Initialize the chunk runnable. @@ -722,6 +732,7 @@ def __init__( the chunk's geometries to the write queue and frees them. analysis_scale: Analysis scale ("regional", "national", or "local") epsg_code: EPSG code for coordinate transformation + h3_resolution: H3 resolution override for regional scale. """ super().__init__() self.chunk = chunk @@ -731,6 +742,7 @@ def __init__( self.write_callback = write_callback self.analysis_scale = analysis_scale self.epsg_code = epsg_code + self.h3_resolution = h3_resolution self.result = None self.error = None self.setAutoDelete(False) # We manage lifecycle manually @@ -743,7 +755,7 @@ def run(self): # Use H3 task for regional scale, regular grid task otherwise if self.analysis_scale == "regional": - h3_res = get_h3_resolution_for_scale(self.analysis_scale) or 6 + h3_res = self.h3_resolution task = GridFromBboxH3Task( index, ( @@ -815,6 +827,9 @@ class StudyAreaProcessingTask(QgsTask): # Signal emitted when GHSL download fails - allows UI to prompt user to continue or abort ghsl_download_failed = pyqtSignal(str) + MIN_H3_ESTIMATED_CELLS = 3 + MAX_H3_ESTIMATED_CELLS = 200000 + # Signal emitted when waiting for user response about GHSL failure ghsl_user_response_ready = pyqtSignal() @@ -827,6 +842,7 @@ def __init__( feedback: "QgsFeedback | None" = None, crs=None, analysis_scale: str = "national", + h3_resolution: int = None, ): """Initialize the study area processing task. @@ -839,6 +855,7 @@ def __init__( crs: Target CRS. If None, a UTM zone will be computed. analysis_scale: Analysis scale ("regional", "national", or "local"). Regional uses H3 hexagonal grids, others use square grids. + h3_resolution: Optional H3 resolution override for regional analysis. Raises: RuntimeError: If the input layer cannot be opened with OGR. @@ -859,18 +876,17 @@ def __init__( Exception: If the CRS is not EPSG-based. """ super().__init__("Study Area Preparation", QgsTask.CanCancel) - - # Configure GDAL for optimized GeoPackage writes - # These settings trade crash safety for performance - acceptable for processing tasks - gdal.SetConfigOption("OGR_SQLITE_JOURNAL", "MEMORY") - gdal.SetConfigOption("OGR_SQLITE_SYNCHRONOUS", "OFF") - gdal.SetConfigOption("SQLITE_USE_OGR_VFS", "YES") - log_message("Using optimized GeoPackage write settings") + self._previous_gdal_sqlite_options = {} self.input_vector_path = self.export_qgs_layer_to_shapefile(layer, working_dir) self.field_name = field_name self.cell_size_m = cell_size_m self.analysis_scale = analysis_scale + self.h3_resolution = ( + h3_resolution if h3_resolution is not None else (get_h3_resolution_for_scale("regional") or 6) + ) + if self.analysis_scale == "regional": + log_message(f"Using H3 resolution: {self.h3_resolution}") self.working_dir = working_dir self.gpkg_path = os.path.join(working_dir, "study_area", "study_area.gpkg") self.counter = 0 @@ -1223,6 +1239,7 @@ def run(self): Returns: True if processing completed successfully, False otherwise. """ + self._set_sqlite_write_safety_options() try: # 1) Create the bounding box as a single polygon feature # and save to GeoPackage @@ -1360,7 +1377,18 @@ def run(self): log_message(f"Areas that could not be processed due to errors: {self.error_count}") log_message(f"Total cells generated: {self.total_cells}") - # 4) Create a VRT of all generated raster masks + # 4) Add model columns to the grid layer + model_path = os.path.join(self.working_dir, "model.json") + if os.path.exists(model_path): + log_message("Adding model columns to study_area_grid layer...") + if add_model_columns_to_grid(self.gpkg_path, model_path): + log_message("Model columns added successfully") + else: + log_message("Failed to add model columns to grid", level="WARNING") + else: + log_message(f"Model file not found at {model_path}, skipping column addition", level="WARNING") + + # 5) Create a VRT of all generated raster masks self.create_raster_vrt() except Exception as e: @@ -1372,11 +1400,31 @@ def run(self): return False finally: + self._restore_sqlite_write_safety_options() # Explicit cleanup of GDAL resources to prevent memory leaks self._cleanup_gdal_resources() return True + def _set_sqlite_write_safety_options(self) -> None: + """Configure safer SQLite options for GeoPackage writes. + + These options are process-wide in GDAL, so we snapshot and restore them + per task execution to avoid leaking settings into workflow processing. + """ + option_keys = ["OGR_SQLITE_JOURNAL", "OGR_SQLITE_SYNCHRONOUS", "SQLITE_USE_OGR_VFS"] + self._previous_gdal_sqlite_options = {key: gdal.GetConfigOption(key) for key in option_keys} + + gdal.SetConfigOption("OGR_SQLITE_JOURNAL", "WAL") + gdal.SetConfigOption("OGR_SQLITE_SYNCHRONOUS", "NORMAL") + gdal.SetConfigOption("SQLITE_USE_OGR_VFS", "YES") + log_message("Configured safer GeoPackage write settings (WAL/NORMAL)") + + def _restore_sqlite_write_safety_options(self) -> None: + """Restore GDAL SQLite options captured before task execution.""" + for key, value in self._previous_gdal_sqlite_options.items(): + gdal.SetConfigOption(key, value) + def _cleanup_gdal_resources(self): """Clean up GDAL/OGR resources to prevent memory leaks and file handle issues.""" try: @@ -1609,6 +1657,13 @@ def process_singlepart_geometry(self, geom, normalized_name, area_name, shared_l intersects_ghsl = self.check_ghsl_intersection(geom) log_message(f"{normalized_name} intersects GHSL: {intersects_ghsl}") + if self.analysis_scale == "regional" and not self._validate_regional_h3_runtime_guard(geom, normalized_name): + self.error_count += 1 + self.counter += 1 + progress = int((self.counter / self.parts_count) * 100) + self.setProgress(progress) + return + # Save the geometry (in the target CRS) to "study_area_polygons" self.save_geometry_to_geopackage("study_area_polygons", geom, normalized_name, intersects_ghsl) self.set_status_tracking_table_value(normalized_name, "geometry_processed", 1) @@ -2260,9 +2315,7 @@ def _create_grid_task(self, index, bbox_chunk, geom, cell_size, feedback): Grid task (GridFromBboxTask or GridFromBboxH3Task) """ if self.analysis_scale == "regional": - h3_res = get_h3_resolution_for_scale(self.analysis_scale) - if h3_res is None: - h3_res = 6 # Default to resolution 6 for regional scale + h3_res = self.h3_resolution log_message(f"Creating H3 grid task (resolution {h3_res}) for chunk {index}") task = GridFromBboxH3Task( index, @@ -2384,6 +2437,7 @@ def _write_callback(task, start_time): write_callback=_write_callback, analysis_scale=self.analysis_scale, epsg_code=self.epsg_code, + h3_resolution=self.h3_resolution, ) runnables.append(runnable) pool.start(runnable) @@ -2407,13 +2461,13 @@ def write_chunk(self, layer, task, normalized_name): task: GridFromBboxTask or GridFromBboxH3Task with generated features normalized_name: Area name for this chunk """ - self.track_time("Preparing chunks", task.run_time) + self.metrics["Preparing chunks"] += task.run_time # Check if this is an H3 task (features are tuples of (h3_index, geometry)) is_h3_task = isinstance(task.features_out[0], tuple) if task.features_out else False if is_h3_task: - h3_resolution = get_h3_resolution_for_scale(self.analysis_scale) + h3_resolution = self.h3_resolution for feature in task.features_out: # Get unique grid_id with lock @@ -2469,7 +2523,7 @@ def create_grid_layer_if_not_exists(self, layer_name): # Add H3 fields for regional scale if self.analysis_scale == "regional": - h3_res = get_h3_resolution_for_scale(self.analysis_scale) + h3_res = self.h3_resolution field_defn = ogr.FieldDefn("h3_index", ogr.OFTString) layer.CreateField(field_defn) field_defn = ogr.FieldDefn("h3_resolution", ogr.OFTInteger) @@ -2698,6 +2752,57 @@ def chunk_bbox(self, xmin, xmax, ymin, ymax, cell_size, chunk_size=1000): log_message(f"Created Chunk bbox: {x_start_coord}, {x_end_coord}, {ymin}, {ymax}") yield (x_start_coord, x_end_coord, y_start_coord, y_end_coord) + def _estimate_geom_area_km2(self, geom): + """Estimate geometry area in square kilometers using Mollweide projection.""" + if geom is None or geom.IsEmpty(): + return 0.0 + + geom_clone = geom.Clone() + geom_clone.AssignSpatialReference(self.target_spatial_ref) + + mollweide_srs = osr.SpatialReference() + mollweide_srs.SetFromUserInput("ESRI:54009") + + transform_to_mollweide = osr.CoordinateTransformation(self.target_spatial_ref, mollweide_srs) + geom_clone.Transform(transform_to_mollweide) + + area_m2 = abs(geom_clone.GetArea()) + return area_m2 / 1000000.0 + + def _validate_regional_h3_runtime_guard(self, geom, normalized_name): + """Fail fast at runtime for unsafe H3 density choices.""" + try: + area_km2 = self._estimate_geom_area_km2(geom) + estimated_cells = estimate_h3_cells_for_area(area_km2, self.h3_resolution) + cell_area_km2 = h3_cell_area_km2(self.h3_resolution) + + if estimated_cells > self.MAX_H3_ESTIMATED_CELLS: + log_message( + ( + f"Skipping {normalized_name}: H3 resolution {self.h3_resolution} is too fine " + f"for runtime safeguards (estimated {estimated_cells:,} cells from {area_km2:,.2f} km2, " + f"cell area {cell_area_km2:,.6f} km2, max {self.MAX_H3_ESTIMATED_CELLS:,})." + ), + level="WARNING", + ) + return False + + if estimated_cells < self.MIN_H3_ESTIMATED_CELLS: + log_message( + ( + f"Skipping {normalized_name}: H3 resolution {self.h3_resolution} is too coarse " + f"for runtime safeguards (estimated {estimated_cells} cells from {area_km2:,.2f} km2, " + f"minimum {self.MIN_H3_ESTIMATED_CELLS})." + ), + level="WARNING", + ) + return False + + return True + except Exception as e: + log_message(f"Runtime H3 safeguard check failed for {normalized_name}: {e}", level="WARNING") + return False + ########################################################################## # Create Raster Mask ########################################################################## diff --git a/geest/core/utilities.py b/geest/core/utilities.py index abc88c08..60f1ca38 100644 --- a/geest/core/utilities.py +++ b/geest/core/utilities.py @@ -177,6 +177,203 @@ def add_to_map( ) +def add_grid_layer_to_map( + item: "JsonTreeItem", + column_name: str, + working_directory: str, + layer_name: str = None, + group: str = "GeoE3", +): + """Add a styled grid layer to the map for a specific column. + + This function creates a layer from study_area_grid and applies the + indicator-vector-template.qml style with the column name substituted. + + Args: + item: The tree item (indicator/factor/dimension) to display. + column_name: The column in study_area_grid to symbolize. + working_directory: Path to the working directory containing study_area.gpkg. + layer_name: Optional display name for the layer. Defaults to item name. + group: The top-level group name. Defaults to "GeoE3". + """ + import tempfile + + from geest.utilities import resources_path + + log_message(f"add_grid_layer_to_map called with column: {column_name}") + log_message(f"Working directory: {working_directory}") + + if not working_directory: + log_message( + "Working directory is not set. Cannot add grid layer.", + tag="GeoE3", + level=Qgis.Warning, + ) + return + + # Construct the GeoPackage path + gpkg_path = os.path.join(working_directory, "study_area", "study_area.gpkg") + if not os.path.exists(gpkg_path): + log_message( + f"GeoPackage not found: {gpkg_path}", + tag="GeoE3", + level=Qgis.Warning, + ) + return + + # Create layer URI for study_area_grid + layer_uri = f"{gpkg_path}|layername=study_area_grid" + + if not layer_name: + layer_name = f"{item.data(0)} (Grid)" + + log_message(f"Adding grid layer for column: {column_name}") + log_message(f"Layer URI: {layer_uri}") + log_message(f"Layer name: {layer_name}") + + # Load the layer + layer = QgsVectorLayer(layer_uri, layer_name, "ogr") + if not layer.isValid(): + log_message( + f"Layer {layer_name} is invalid and cannot be added.", + tag="GeoE3", + level=Qgis.Warning, + ) + return + + # Verify the column exists in the layer + field_names = [field.name() for field in layer.fields()] + log_message(f"Available columns: {field_names[:10]}...") # Log first 10 + if column_name not in field_names: + log_message( + f"Column '{column_name}' not found in study_area_grid. Available columns: {field_names}", + tag="GeoE3", + level=Qgis.Warning, + ) + return + + # Apply filter to exclude NULL values for the column being visualized + filter_expression = f'"{column_name}" IS NOT NULL' + layer.setSubsetString(filter_expression) + log_message(f"Applied filter: {filter_expression}") + + # Load the QML template and substitute the column name + template_path = resources_path("resources", "qml", "indicator-vector-template.qml") + log_message(f"Template path: {template_path}") + if not os.path.exists(template_path): + log_message( + f"QML template not found: {template_path}", + tag="GeoE3", + level=Qgis.Warning, + ) + return + + with open(template_path, "r") as f: + qml_content = f.read() + + # Replace the [attribute] placeholder with the actual column name + qml_content = qml_content.replace("[attribute]", column_name) + log_message(f"Substituted column '{column_name}' in QML template") + + # Write to a temporary file and apply the style + with tempfile.NamedTemporaryFile(mode="w", suffix=".qml", delete=False) as tmp: + tmp.write(qml_content) + tmp_path = tmp.name + + try: + result = layer.loadNamedStyle(tmp_path) + if not result[0]: + log_message( + f"Failed to apply style: {result[1]}", + tag="GeoE3", + level=Qgis.Warning, + ) + finally: + # Clean up temp file + try: + os.unlink(tmp_path) + except OSError: + pass + + project = QgsProject.instance() + + # Check if 'GeoE3' group exists, otherwise create it + root = project.layerTreeRoot() + geoe3_group = root.findGroup(group) + if geoe3_group is None: + geoe3_group = root.insertGroup(0, group) + geoe3_group.setIsMutuallyExclusive(True) + + # Traverse the tree view structure to determine the appropriate subgroup + path_list = item.getPaths() + parent_group = geoe3_group + # Truncate the last item from the path list + path_list = path_list[:-1] + + for path in path_list: + sub_group = parent_group.findGroup(path) + if sub_group is None: + sub_group = parent_group.addGroup(path) + sub_group.setIsMutuallyExclusive(True) + parent_group = sub_group + + # Check if a layer with the same name exists in the group + existing_layer = None + layer_tree_layer = None + for child in parent_group.children(): + if isinstance(child, QgsLayerTreeGroup): + continue + if child.layer().name() == layer_name: + existing_layer = child.layer() + layer_tree_layer = child + break + + # If the layer exists, update its style and filter instead of re-adding + if existing_layer is not None: + log_message(f"Refreshing existing layer: {existing_layer.name()}") + # Update filter for the column being visualized + existing_layer.setSubsetString(filter_expression) + # Re-apply style to existing layer + with tempfile.NamedTemporaryFile(mode="w", suffix=".qml", delete=False) as tmp: + tmp.write(qml_content) + tmp_path = tmp.name + try: + existing_layer.loadNamedStyle(tmp_path) + finally: + try: + os.unlink(tmp_path) + except OSError: + pass + layer_tree_layer.setItemVisibilityChecked(True) + existing_layer.triggerRepaint() + else: + # Add the new layer + QgsProject.instance().addMapLayer(layer, False) + layer_tree_layer = parent_group.addLayer(layer) + layer_tree_layer.setExpanded(False) + log_message(f"Added layer: {layer.name()} to group: {parent_group.name()}") + + # Ensure the layer and its parent groups are visible + current_group = parent_group + while current_group is not None: + current_group.setExpanded(True) + current_group.setItemVisibilityChecked(True) + current_group = current_group.parent() + + layer_tree_layer.setItemVisibilityChecked(True) + + # Refresh the canvas + repaint_layer = existing_layer if existing_layer is not None else layer + repaint_layer.triggerRepaint() + iface.mapCanvas().refresh() + + log_message( + f"Grid layer {layer_name} for column {column_name} added to map.", + tag="GeoE3", + level=Qgis.Info, + ) + + def validate_network_layer(layer_path: str, expected_crs: QgsCoordinateReferenceSystem) -> tuple: """Validate network layer for road network analysis. diff --git a/geest/core/women_considerations.py b/geest/core/women_considerations.py new file mode 100644 index 00000000..c9bccc57 --- /dev/null +++ b/geest/core/women_considerations.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +"""Helpers for women considerations factor enablement rules.""" + +__copyright__ = "Copyright 2024, Tim Sutton" +__license__ = "GPL version 3" +__email__ = "tim@kartoza.com" +__revision__ = "$Format:%H$" + + +def resolve_women_enabling_for_factor(factor_id: str, women_enabling: int) -> int: + """Resolve women-enabling value with backward-compatibility rules. + + Args: + factor_id: Factor identifier from the model. + women_enabling: Original women-enabling value from model metadata. + + Returns: + Effective women-enabling value to use for enable/disable logic. + """ + if factor_id.lower() == "education" and women_enabling == 0: + return 1 + return women_enabling diff --git a/geest/core/workflows/acled_impact_workflow.py b/geest/core/workflows/acled_impact_workflow.py index 8fa3e749..c92b0019 100644 --- a/geest/core/workflows/acled_impact_workflow.py +++ b/geest/core/workflows/acled_impact_workflow.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- """📦 Acled Impact Workflow module. - This module contains functionality for acled impact workflow. """ - import csv import os @@ -81,33 +79,28 @@ def _process_features_for_area( current_bbox: QgsGeometry, area_features: QgsVectorLayer, index: int, + area_name: str = None, ) -> str: """ Executes the actual workflow logic for a single area Must be implemented by subclasses. - :current_area: Current polygon from our study area. :current_bbox: Bounding box of the above area. :area_features: A vector layer of features to analyse that includes only features in the study area. :index: Iteration / number of area being processed. - :return: Raster file path of the output. """ - # Step 1: Buffer the selected features by relevant # distance for each event type and assign values # to the buffer layer buffered_layer = self._buffer_features(area_features) self.feedback.setProgress(10.0) - # Step 2: Assign values based on event_type # scored_layer = self._assign_scores(buffered_layer) self.feedback.setProgress(40.0) - # Step 3: Dissolve and remove overlapping areas, keeping areas with the lowest value dissolved_layer = self._overlay_analysis(buffered_layer) self.feedback.setProgress(60.0) - # Step 4: Rasterize the dissolved layer raster_output = self._rasterize( dissolved_layer, @@ -117,7 +110,6 @@ def _process_features_for_area( default_value=5, ) self.feedback.setProgress(80.0) - return raster_output def _load_csv_as_point_layer(self) -> QgsVectorLayer: @@ -125,7 +117,6 @@ def _load_csv_as_point_layer(self) -> QgsVectorLayer: Load the CSV file, extract relevant columns (latitude, longitude, event_type), create a point layer from the retained columns, reproject the points to match the CRS of the layers from the GeoPackage, and save the result as a shapefile. - Returns: QgsVectorLayer: The reprojected point layer created from the CSV. """ @@ -133,20 +124,17 @@ def _load_csv_as_point_layer(self) -> QgsVectorLayer: # Set up a coordinate transform from WGS84 to the target CRS transform_context = self.context.project().transformContext() coordinate_transform = QgsCoordinateTransform(source_crs, self.target_crs, transform_context) - # Define fields for the point layer fields = QgsFields() fields.append(QgsField("event_type", QVariant.String)) fields.append(QgsField("value", QVariant.Int)) fields.append(QgsField("buffer_m", QVariant.Int)) fields.append(QgsField("score", QVariant.Int)) - # Create an in-memory point layer in the target CRS point_layer = QgsVectorLayer(f"Point?crs={self.target_crs.authid()}", "acled_points", "memory") point_provider = point_layer.dataProvider() point_provider.addAttributes(fields) # type: ignore point_layer.updateFields() - # Read the CSV and add reprojected points to the layer with open(self.csv_file, newline="", encoding="utf-8") as csvfile: reader = csv.DictReader(csvfile) @@ -155,11 +143,9 @@ def _load_csv_as_point_layer(self) -> QgsVectorLayer: lat = float(row["latitude"]) lon = float(row["longitude"]) event_type = row["event_type"] - # Transform point to the target CRS point_wgs84 = QgsPointXY(lon, lat) point_transformed = coordinate_transform.transform(point_wgs84) - feature = QgsFeature() feature.setGeometry(QgsGeometry.fromPointXY(point_transformed)) value = self.event_scores.get(event_type, 5) @@ -167,7 +153,6 @@ def _load_csv_as_point_layer(self) -> QgsVectorLayer: score = 0 # this will be replaced later with the lowest overlapping score feature.setAttributes([event_type, value, buffer_m, score]) features.append(feature) - point_provider.addFeatures(features) # type: ignore log_message(f"Loaded {len(features)} points from CSV") # Save the layer to disk as a shapefile @@ -179,32 +164,26 @@ def _load_csv_as_point_layer(self) -> QgsVectorLayer: error = QgsVectorFileWriter.writeAsVectorFormat( point_layer, shapefile_path, "utf-8", self.target_crs, "ESRI Shapefile" ) - if error[0] != 0: raise QgsProcessingException(f"Error saving point layer to disk: {error[1]}") - log_message( f"Point layer created from CSV saved to {shapefile_path}", tag="GeoE3", level=Qgis.Info, ) - # Reload the saved shapefile as the final point layer to ensure consistency saved_layer = QgsVectorLayer(shapefile_path, "acled_points", "ogr") if not saved_layer.isValid(): raise QgsProcessingException(f"Failed to reload saved point layer from {shapefile_path}") - return saved_layer def _buffer_features(self, layer: QgsVectorLayer) -> QgsVectorLayer: """ Buffer the input features by 5 km. - Args: layer (QgsVectorLayer): The input feature layer. This layer should be a point layer with two columns: value and buffer_m representing the geoe3 score for the event and the distance to buffer in m. - Returns: QgsVectorLayer: The buffered features layer. """ @@ -285,58 +264,47 @@ def _overlay_analysis(self, input_layer): """ Perform an overlay analysis on a set of circular polygons, prioritizing areas with the lowest value in overlapping regions, and save the result as a shapefile. - This function processes an input shapefile containing circular polygons, each with a value between 1 and 4, representing different priority levels. The function performs an overlay analysis where the polygons overlap and ensures that for any overlapping areas, the polygon with the lowest value (i.e., highest priority) is retained, while polygons with higher values are removed from those regions. - The analysis is performed as follows: 1. The input layer is loaded from the provided shapefile path. 2. A dissolve operation is performed on the input layer to combine any adjacent polygons with the same value. 3. A union operation is performed on the input layer to break the polygons into distinct, non-overlapping areas. 4. For each distinct area, the value from the overlapping polygons is compared, and the minimum value (representing the highest priority) is assigned to that area. 5. The resulting dataset, which consists of non-overlapping polygons with the highest priority (smallest value), is saved to a new shapefile at the specified output path. - Parameters: ----------- input_layer : QgsVectorLayer The input shapefile containing the circular polygons with values between 1 and 4. - output_filepath : str The file path where the output shapefile with the results of the overlay analysis will be saved. The output will be saved in self.workflow_directory. - Returns: -------- None The function does not return a value but writes the result to the specified output shapefile. - Logging: -------- Messages related to the status of the operation (success or failure) are logged using QgsMessageLog with the tag 'GeoE3' and the log level set to Qgis.Info. - Raises: ------- IOError: If the input layer cannot be loaded or if an error occurs during the file writing process. - Example: -------- To perform an overlay analysis on a shapefile located at "path/to/input.shp" and save the result to "path/to/output.shp", use the following: - overlay_analysis(qgis_vector_layer) """ log_message("Overlay analysis started") # Step 1: Load the input layer from the provided shapefile path # layer = QgsVectorLayer(input_filepath, "circles_layer", "ogr") - if not input_layer.isValid(): log_message("Layer failed to load!") return - # Step 2: Perform the dissolve operation to separate disjoint polygons dissolve_output_path = os.path.join(self.workflow_directory, f"{self.layer_id}_dissolve.shp") dissolve = processing.run( # type: ignore[index] @@ -375,16 +343,13 @@ def _overlay_analysis(self, input_layer): log_message(f"Input layer field types: {[field.typeName() for field in union.fields()]}") # Step 4: Iterate through the unioned features to assign the minimum value in overlapping areas unique_geometries = {} - for feature in union.getFeatures(): geom = feature.geometry().asWkt() attrs = feature.attributes() # Use geometry as a key to identify unique areas value = attrs[union.fields().indexFromName("value")] - log_message( f"Processing feature with min value: {value}", ) - # Check if this geometry is already in the dictionary if geom in unique_geometries: # If it exists, update only if the new min_value is lower @@ -396,19 +361,16 @@ def _overlay_analysis(self, input_layer): new_feature.setGeometry(feature.geometry()) new_feature.setAttributes([value]) unique_geometries[geom] = new_feature - # Step 5: Create a memory layer to store the result result_layer = QgsVectorLayer("Polygon", "result_layer", "memory") result_layer.setCrs(self.target_crs) provider = result_layer.dataProvider() - # Step 6: Add a field to store the minimum value (lower number = higher rank) provider.addAttributes([QgsField("min_value", QVariant.Int)]) result_layer.updateFields() # Step 7: Add the filtered features to the result layer for unique_feature in unique_geometries.values(): provider.addFeature(unique_feature) - full_output_filepath = os.path.join(self.workflow_directory, f"{self.layer_id}_final.shp") # Step 8: Save the result layer to the specified output shapefile error = QgsVectorFileWriter.writeAsVectorFormat( @@ -418,7 +380,6 @@ def _overlay_analysis(self, input_layer): result_layer.crs(), "ESRI Shapefile", ) - if error[0] == 0: log_message( f"Overlay analysis complete, output saved to {full_output_filepath}", @@ -437,16 +398,15 @@ def _process_raster_for_area( current_bbox: QgsGeometry, area_raster: str, index: int, + area_name: str = None, ): """ Executes the actual workflow logic for a single area using a raster. - :current_area: Current polygon from our study area. :clip_area: Polygon to clip the raster to which is aligned to cell edges. :current_bbox: Bounding box of the above area. :area_raster: A raster layer of features to analyse that includes only bbox pixels in the study area. :index: Index of the current area. - :return: Path to the reclassified raster. """ pass @@ -457,6 +417,7 @@ def _process_aggregate_for_area( clip_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: str = None, ): """ Executes the workflow, reporting progress through the feedback object and checking for cancellation. diff --git a/geest/core/workflows/aggregation_workflow_base.py b/geest/core/workflows/aggregation_workflow_base.py index ce3730b5..7c5a7d48 100644 --- a/geest/core/workflows/aggregation_workflow_base.py +++ b/geest/core/workflows/aggregation_workflow_base.py @@ -2,9 +2,14 @@ """📦 Aggregation Workflow Base module. This module contains functionality for aggregation workflow base. + +Supports both raster-first (legacy) and grid-first aggregation approaches. +The grid-first approach writes aggregated values directly to study_area_grid +columns, then optionally rasterizes from the grid. """ import os +from typing import Dict, Optional from qgis.analysis import QgsRasterCalculator, QgsRasterCalculatorEntry from qgis.core import ( @@ -16,6 +21,11 @@ ) from geest.core import JsonTreeItem +from geest.core.grid_column_utils import ( + clear_grid_column, + rasterize_grid_column, + write_aggregation_to_grid, +) from geest.utilities import log_message from .workflow_base import WorkflowBase @@ -51,6 +61,9 @@ def __init__( self.id = None # This should be set by the child class self.weight_key = None # This should be set by the child class self.aggregation = True + # Grid-first mode: write results to grid columns first, then rasterize + # Set to True to use the new grid-first approach + self.use_grid_first = True self.feedback.setProgress(10.0) def aggregate(self, input_files: list, index: int) -> str: @@ -247,26 +260,256 @@ def get_raster_dict(self, index) -> list: ) return raster_files + def get_grid_columns_and_weights(self) -> Dict[str, float]: + """Get the list of grid columns and weights for aggregation. + + This is the grid-first alternative to get_raster_dict(). Instead of + returning raster file paths, it returns column names from study_area_grid. + + Returns: + Dict mapping column names to their weights. + Example: {"indicator1": 0.3, "indicator2": 0.3, "indicator3": 0.4} + """ + columns_weights = {} + if self.guids is None: + raise ValueError("No GUIDs provided for aggregation") + + for guid in self.guids: + item = self.item.getItemByGuid(guid) + status = item.getStatus() == "Completed successfully" + mode = item.attributes().get("analysis_mode", "Do Not Use") == "Do Not Use" + excluded = item.getStatus() == "Excluded from analysis" + disabled = not item.is_enabled() + raw_id = item.attribute("id").lower().replace(" ", "_").replace("-", "_") + # Add prefix based on item role to match column naming + item_role = item.role if hasattr(item, "role") else "" + if item_role == "dimension": + item_id = f"dim_{raw_id}" + elif item_role == "factor": + item_id = f"fac_{raw_id}" + else: + item_id = raw_id # indicators keep raw ID + + if not status and not mode and not excluded and not disabled: + raise ValueError( + f"{item_id} is not completed successfully and is not set to 'Do Not Use' or 'Excluded from analysis'" + ) + + if mode: + log_message( + f"Skipping {item.attribute('id')} as it is set to 'Do Not Use'", + tag="GeoE3", + level=Qgis.Info, + ) + continue + if excluded: + log_message( + f"Skipping {item.attribute('id')} as it is excluded from analysis", + tag="GeoE3", + level=Qgis.Info, + ) + continue + if disabled: + log_message( + f"Skipping {item.attribute('id')} as it is disabled (women considerations)", + tag="GeoE3", + level=Qgis.Info, + ) + continue + + # Get weight for this item + weight = item.attribute(self.weight_key, "") + try: + weight = float(weight) + except (ValueError, TypeError): + weight = 1.0 # Default fallback to 1.0 if weight is invalid + + # Column name is the sanitized item ID + column_name = item_id[:63] # Match sanitization in grid_column_utils + columns_weights[column_name] = weight + + log_message(f"Adding column: {column_name} with weight: {weight}") + + log_message( + f"Total columns found for aggregation: {len(columns_weights)}", + tag="GeoE3", + level=Qgis.Info, + ) + return columns_weights + + def aggregate_grid(self, area_name: str) -> int: + """Perform weighted aggregation directly on grid columns. + + This is the grid-first alternative to aggregate(). Instead of using + QgsRasterCalculator on raster files, it uses SQL to aggregate values + directly in the study_area_grid table. + + Args: + area_name: The name of the area being processed. + + Returns: + Number of cells updated, or -1 on error. + """ + columns_weights = self.get_grid_columns_and_weights() + + if not columns_weights: + log_message( + "Error: Found no columns to aggregate.", + tag="GeoE3", + level=Qgis.Warning, + ) + return -1 + + log_message( + f"Aggregating {len(columns_weights)} columns into {self.layer_id} for area {area_name}", + tag="GeoE3", + level=Qgis.Info, + ) + + # Clear stale values before writing new aggregation + clear_grid_column(self.gpkg_path, self.layer_id) + + # Use the grid-first aggregation function + updated_count = write_aggregation_to_grid( + gpkg_path=self.gpkg_path, + target_column=self.layer_id, + source_columns_weights=columns_weights, + area_name=area_name, + use_coalesce=True, + ) + + if updated_count >= 0: + log_message( + f"Grid aggregation completed: updated {updated_count} cells for {self.layer_id}", + tag="GeoE3", + level=Qgis.Info, + ) + else: + log_message( + f"Grid aggregation failed for {self.layer_id}", + tag="GeoE3", + level=Qgis.Warning, + ) + + return updated_count + + def rasterize_from_grid( + self, + area_name: str, + bbox: QgsGeometry, + index: int, + ) -> Optional[str]: + """Rasterize the grid column to create a raster output. + + This creates a raster from the aggregated grid column using gdal_rasterize. + + Args: + area_name: The name of the area being processed. + bbox: Bounding box geometry for the output raster extent. + index: The index of the area being processed. + + Returns: + Path to the output raster, or None on error. + """ + output_path = os.path.join( + self.workflow_directory, + f"{self.layer_id}_aggregated_{index}.tif", + ) + + # Get extent from bbox + rect = bbox.boundingBox() + extent = (rect.xMinimum(), rect.yMinimum(), rect.xMaximum(), rect.yMaximum()) + + success = rasterize_grid_column( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + output_raster_path=output_path, + cell_size=self.cell_size_m, + extent=extent, + nodata=-9999.0, + area_name=area_name, + ) + + if success: + log_message( + f"Rasterized grid column {self.layer_id} to {output_path}", + tag="GeoE3", + level=Qgis.Info, + ) + # Write the output path to attributes + self.attributes[self.result_file_key] = output_path + return output_path + else: + log_message( + f"Failed to rasterize grid column {self.layer_id}", + tag="GeoE3", + level=Qgis.Warning, + ) + return None + def _process_aggregate_for_area( self, current_area: QgsGeometry, clip_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: Optional[str] = None, ): - """ - Executes the workflow, reporting progress through the feedback object and checking for cancellation. - """ - _ = current_area # Unused in this analysis - _ = clip_area # Unused in this analysis - _ = current_bbox # Unused in this analysis + """Execute aggregation workflow for a single area. + + Supports both raster-first (legacy) and grid-first aggregation modes. + The mode is controlled by self.use_grid_first flag. + + Args: + current_area: Current polygon from our study area. + clip_area: Polygon to clip the raster to which is aligned to cell edges. + current_bbox: Bounding box of the above area. + index: Index of the current area. + area_name: Name of the area being processed (for grid-first mode). + Returns: + Path to the aggregated raster file, or None on error. + """ # Log the execution log_message( - f"Executing {self.analysis_mode} Aggregation Workflow", + f"Executing {self.analysis_mode} Aggregation Workflow (grid_first={self.use_grid_first})", tag="GeoE3", level=Qgis.Info, ) + + if self.use_grid_first: + # Grid-first mode: aggregate directly in grid columns + return self._process_aggregate_grid_first( + current_area=current_area, + clip_area=clip_area, + current_bbox=current_bbox, + index=index, + area_name=area_name, + ) + else: + # Legacy raster-first mode + return self._process_aggregate_raster_first( + current_area=current_area, + clip_area=clip_area, + current_bbox=current_bbox, + index=index, + ) + + def _process_aggregate_raster_first( + self, + current_area: QgsGeometry, + clip_area: QgsGeometry, + current_bbox: QgsGeometry, + index: int, + ) -> Optional[str]: + """Legacy raster-first aggregation. + + Uses QgsRasterCalculator to aggregate raster files. + """ + _ = current_area # Unused + _ = clip_area # Unused + _ = current_bbox # Unused + raster_files = self.get_raster_dict(index) if not raster_files or not isinstance(raster_files, dict): @@ -281,13 +524,77 @@ def _process_aggregate_for_area( return None log_message( - f"Found {len(raster_files)} raster files in 'Result File'. Proceeding with aggregation.", + f"Found {len(raster_files)} raster files in 'Result File'. Proceeding with raster aggregation.", tag="GeoE3", level=Qgis.Info, ) - # Perform aggregation only if raster files are provided + # Perform aggregation using raster calculator result_file = self.aggregate(raster_files, index) + return result_file + + def _process_aggregate_grid_first( + self, + current_area: QgsGeometry, + clip_area: QgsGeometry, + current_bbox: QgsGeometry, + index: int, + area_name: Optional[str] = None, + ) -> Optional[str]: + """Grid-first aggregation. + + Aggregates values directly in grid columns using SQL, then + optionally rasterizes from the grid. + """ + _ = current_area # Unused + + # Step 1: Aggregate grid columns + try: + columns_weights = self.get_grid_columns_and_weights() + except ValueError as e: + error = str(e) + log_message( + error, + tag="GeoE3", + level=Qgis.Warning, + ) + self.attributes[self.result_key] = f"{self.analysis_mode} Aggregation Workflow Skipped" + self.attributes["error"] = error + return None + + if not columns_weights: + error = "No valid columns found for aggregation. Cannot proceed (likely all factors disabled or excluded)." + log_message( + error, + tag="GeoE3", + level=Qgis.Warning, + ) + self.attributes[self.result_key] = f"{self.analysis_mode} Aggregation Workflow Skipped" + self.attributes["error"] = error + return None + + log_message( + f"Found {len(columns_weights)} columns for grid aggregation: {list(columns_weights.keys())}", + tag="GeoE3", + level=Qgis.Info, + ) + + # Perform SQL aggregation on grid + updated_count = self.aggregate_grid(area_name) + if updated_count < 0: + log_message( + f"Grid aggregation failed for area {area_name}", + tag="GeoE3", + level=Qgis.Warning, + ) + return None + + # Step 2: Rasterize from grid column for VRT generation + result_file = self.rasterize_from_grid( + area_name=area_name, + bbox=current_bbox, + index=index, + ) return result_file diff --git a/geest/core/workflows/analysis_aggregation_workflow.py b/geest/core/workflows/analysis_aggregation_workflow.py index d7d1a2d1..9d9bf6f8 100644 --- a/geest/core/workflows/analysis_aggregation_workflow.py +++ b/geest/core/workflows/analysis_aggregation_workflow.py @@ -47,7 +47,7 @@ def __init__( self.id = ( self.item.attribute("analysis_name").lower().replace(" ", "_").replace("'", "") ) # should not be needed any more - self.layer_id = "geoe3" + self.layer_id = "geoe3" # Must match column name in grid_column_utils.get_aggregate_column_names() self.weight_key = "analysis_weighting" self.workflow_name = "analysis_aggregation" # Override the default working directory defined in the base class diff --git a/geest/core/workflows/classified_polygon_workflow.py b/geest/core/workflows/classified_polygon_workflow.py index 5aaf26c0..62fa1f91 100644 --- a/geest/core/workflows/classified_polygon_workflow.py +++ b/geest/core/workflows/classified_polygon_workflow.py @@ -1,10 +1,16 @@ # -*- coding: utf-8 -*- -from urllib.parse import unquote - """📦 Classified Polygon Workflow module. This module contains functionality for classified polygon workflow. + +Supports grid-first mode where polygon classification scores are written +directly to the study_area_grid column, then rasterized. """ + +import os +from typing import Optional +from urllib.parse import unquote + from qgis.core import ( Qgis, QgsFeedback, @@ -17,6 +23,11 @@ from qgis.PyQt.QtCore import QVariant from geest.core import JsonTreeItem +from geest.core.grid_column_utils import ( + clear_grid_column, + rasterize_grid_column, + write_buffer_values_to_grid, +) from geest.utilities import log_message from .workflow_base import WorkflowBase @@ -38,6 +49,7 @@ def __init__( ): """ Initialize the workflow with attributes and feedback. + :param attributes: Item containing workflow parameters. :param feedback: QgsFeedback object for progress reporting and cancellation. :context: QgsProcessingContext object for processing. This can be used to pass objects to the thread. e.g. the QgsProject Instance @@ -50,7 +62,6 @@ def __init__( layer_path = self.attributes.get("classify_polygon_into_classes_shapefile", None) if layer_path: layer_path = unquote(layer_path) - if not layer_path: log_message( "Invalid layer found in use_classify_polygon_into_classes_shapefile, trying use_classify_polygon_into_classes_source.", @@ -65,10 +76,13 @@ def __init__( level=Qgis.Warning, ) return False - self.features_layer = QgsVectorLayer(layer_path, "features_layer", "ogr") - self.selected_field = self.attributes.get("classify_polygon_into_classes_selected_field", "") + self.workflow_name = "classified_polygon" + # Grid-first mode: write results to grid columns first, then rasterize + self.use_grid_first = True + # Track if we've cleared the column (only do once, not per area) + self._column_cleared = False def _process_features_for_area( self, @@ -77,17 +91,21 @@ def _process_features_for_area( current_bbox: QgsGeometry, area_features: QgsVectorLayer, index: int, + area_name: Optional[str] = None, ) -> str: """ - Executes the actual workflow logic for a single area - Must be implemented by subclasses. + Executes the actual workflow logic for a single area. + + Supports grid-first mode where classification scores are written + directly to study_area_grid. :current_area: Current polygon from our study area. :current_bbox: Bounding box of the above area. :area_features: A vector layer of features to analyse that includes only features in the study area. :index: Iteration / number of area being processed. + :area_name: Name of the area being processed. - :return: A raster layer file path if processing completes successfully, False if canceled or failed. + :return: A raster layer file path if processing completes successfully. """ area_features_count = area_features.featureCount() log_message( @@ -95,9 +113,30 @@ def _process_features_for_area( tag="GeoE3", level=Qgis.Info, ) + + if self.use_grid_first: + return self._process_grid_first( + current_bbox=current_bbox, + area_features=area_features, + index=index, + area_name=area_name, + ) + else: + return self._process_raster_first( + current_bbox=current_bbox, + area_features=area_features, + index=index, + ) + + def _process_raster_first( + self, + current_bbox: QgsGeometry, + area_features: QgsVectorLayer, + index: int, + ) -> str: + """Legacy raster-first processing.""" # Step 1: Assign reclassification values based on perceived safety reclassified_layer = self._assign_reclassification_to_safety(area_features) - # Step 2: Rasterize the data raster_output = self._rasterize( reclassified_layer, @@ -108,6 +147,64 @@ def _process_features_for_area( ) return raster_output + def _process_grid_first( + self, + current_bbox: QgsGeometry, + area_features: QgsVectorLayer, + index: int, + area_name: str, + ) -> str: + """Grid-first processing - writes classification scores directly to study_area_grid.""" + # Clear column once at the start (not per area) + if not self._column_cleared: + log_message(f"Clearing column {self.layer_id} before processing") + clear_grid_column(self.gpkg_path, self.layer_id) + self._column_cleared = True + + self.progressChanged.emit(10.0) + + # Step 1: Assign reclassification values (scale 0-100 to 0-5) + log_message(f"Scaling classification values for {area_features.featureCount()} polygons") + reclassified_layer = self._assign_reclassification_to_safety(area_features) + + self.progressChanged.emit(40.0) + + # Step 2: Write polygon scores to grid cells + log_message(f"Writing classification scores to grid column {self.layer_id}") + write_buffer_values_to_grid( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + buffer_layer=reclassified_layer, + value_field="value", + aggregation_method="MAX", + feedback=self.feedback, + ) + + self.progressChanged.emit(70.0) + + # Step 3: Rasterize from grid column + output_path = os.path.join( + self.workflow_directory, + f"{self.layer_id}_{index}.tif", + ) + + rect = current_bbox.boundingBox() + extent = (rect.xMinimum(), rect.yMinimum(), rect.xMaximum(), rect.yMaximum()) + + rasterize_grid_column( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + output_raster_path=output_path, + cell_size=self.cell_size_m, + extent=extent, + nodata=-9999.0, + area_name=area_name, + ) + + self.progressChanged.emit(100.0) + log_message(f"Rasterized grid column to {output_path}") + return output_path + def _assign_reclassification_to_safety(self, layer: QgsVectorLayer) -> QgsVectorLayer: """ Assign reclassification values to polygons based on thresholds. @@ -157,6 +254,7 @@ def _process_raster_for_area( current_bbox: QgsGeometry, area_raster: str, index: int, + area_name: str = None, ): """ Executes the actual workflow logic for a single area using a raster. @@ -177,6 +275,7 @@ def _process_aggregate_for_area( clip_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: str = None, ): """ Executes the workflow, reporting progress through the feedback object and checking for cancellation. diff --git a/geest/core/workflows/contextual_index_score_workflow.py b/geest/core/workflows/contextual_index_score_workflow.py index 6292611d..96b733e1 100644 --- a/geest/core/workflows/contextual_index_score_workflow.py +++ b/geest/core/workflows/contextual_index_score_workflow.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- - """ Specialised index score workflow for use in contextual dimensions. -""" +Supports grid-first mode where the index score is written directly to +the study_area_grid column, then optionally rasterized. +""" import os +from typing import Optional from qgis import processing # noqa: F401 # QGIS processing toolbox from qgis.core import ( # noqa: F401 @@ -20,6 +22,10 @@ from qgis.PyQt.QtCore import QVariant from geest.core import JsonTreeItem # noqa: unused F401 +from geest.core.grid_column_utils import ( + rasterize_grid_column, + write_uniform_value_to_grid, +) from geest.utilities import log_message from .contextual_index_score_mappings import score_mapping @@ -52,11 +58,9 @@ def __init__( super().__init__( item, cell_size_m, analysis_scale, feedback, context, working_directory ) # ⭐️ Item is a reference - whatever you change in this item will directly update the tree - index_score = self.attributes.get("index_score", 0) log_message(f"Index score before rescaling to contextual scale: {index_score}") # Define mapping rules as (min_score, output_score) pairs - # Find the highest threshold less than or equal to index_score for threshold in sorted(score_mapping.keys(), reverse=True): if index_score >= threshold: @@ -68,6 +72,8 @@ def __init__( True # Normally we would set this to a QgsVectorLayer but in this workflow it is not needed ) self.workflow_name = "contextual_index_score" + # Grid-first mode: write results to grid columns first, then rasterize + self.use_grid_first = True def _process_features_for_area( self, @@ -76,31 +82,50 @@ def _process_features_for_area( current_bbox: QgsGeometry, area_features: QgsVectorLayer, index: int, + area_name: Optional[str] = None, ) -> str: """ - Executes the actual workflow logic for a single area - Must be implemented by subclasses. + Executes the actual workflow logic for a single area. + + Supports both raster-first (legacy) and grid-first modes. :current_area: Current polygon from our study area. :current_bbox: Bounding box of the above area. - :area_features: A vector layer of features to analyse that includes only features in the study area. + :area_features: A vector layer of features to analyse (unused for index_score). :index: Iteration / number of area being processed. - + :area_name: Name of the area being processed (for grid-first mode). :return: Raster file path of the output. """ _ = area_features # unused - log_message(f"Processing area {index} score workflow") - + log_message(f"Processing area {index} contextual score workflow (grid_first={self.use_grid_first})") log_message(f"Index score: {self.index_score}") - self.progressChanged.emit(10.0) # We just use nominal intervals for progress updates - - # Create a scored boundary layer filtered by current_area + self.progressChanged.emit(10.0) + + if self.use_grid_first and area_name: + return self._process_grid_first( + current_bbox=current_bbox, + index=index, + area_name=area_name, + ) + else: + return self._process_raster_first( + clip_area=clip_area, + current_bbox=current_bbox, + index=index, + ) + + def _process_raster_first( + self, + clip_area: QgsGeometry, + current_bbox: QgsGeometry, + index: int, + ) -> str: + """Legacy raster-first processing.""" scored_layer = self.create_scored_boundary_layer( clip_area=clip_area, index=index, ) - self.progressChanged.emit(30.0) # We just use nominal intervals for progress updates - # Create a scored boundary layer + self.progressChanged.emit(30.0) raster_output = self._rasterize( scored_layer, current_bbox, @@ -108,21 +133,59 @@ def _process_features_for_area( value_field="score", default_value=255, ) - self.progressChanged.emit(100.0) # We just use nominal intervals for progress updates - + self.progressChanged.emit(100.0) log_message(f"Raster output: {raster_output}") log_message(f"Workflow completed for area {index}") return raster_output + def _process_grid_first( + self, + current_bbox: QgsGeometry, + index: int, + area_name: str, + ) -> str: + """Grid-first processing - writes to grid column, then rasterizes.""" + self.progressChanged.emit(20.0) + log_message(f"Writing contextual index score {self.index_score} to grid column {self.layer_id}") + + write_uniform_value_to_grid( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + value=self.index_score, + ) + + self.progressChanged.emit(50.0) + + # Rasterize from grid column + output_path = os.path.join( + self.workflow_directory, + f"{self.layer_id}_{index}.tif", + ) + + rect = current_bbox.boundingBox() + extent = (rect.xMinimum(), rect.yMinimum(), rect.xMaximum(), rect.yMaximum()) + + rasterize_grid_column( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + output_raster_path=output_path, + cell_size=self.cell_size_m, + extent=extent, + nodata=-9999.0, + area_name=area_name, + ) + + self.progressChanged.emit(100.0) + log_message(f"Rasterized grid column to {output_path}") + return output_path + def create_scored_boundary_layer(self, clip_area: QgsGeometry, index: int) -> QgsVectorLayer: """ Create a scored boundary layer, filtering features by the current_area. - :param index: The index of the current processing area. :return: A vector layer with a 'score' attribute. """ output_prefix = f"{self.layer_id}_area_{index}" - self.progressChanged.emit(20.0) # We just use nominal intervals for progress updates # Create a new memory layer with the target CRS (EPSG:4326) subset_layer = QgsVectorLayer("Polygon", "subset", "memory") @@ -134,7 +197,6 @@ def create_scored_boundary_layer(self, clip_area: QgsGeometry, index: int) -> Qg subset_layer_data.addAttributes(fields) subset_layer.updateFields() self.progressChanged.emit(40.0) # We just use nominal intervals for progress updates - feature = QgsFeature(subset_layer.fields()) feature.setGeometry(clip_area) score_field_index = subset_layer.fields().indexFromName("score") @@ -144,7 +206,6 @@ def create_scored_boundary_layer(self, clip_area: QgsGeometry, index: int) -> Qg subset_layer_data.addFeatures(features) subset_layer.commitChanges() self.progressChanged.emit(60.0) # We just use nominal intervals for progress updates - shapefile_path = os.path.join(self.workflow_directory, f"{output_prefix}.shp") # Use QgsVectorFileWriter to save the layer to a shapefile QgsVectorFileWriter.writeAsVectorFormat( @@ -156,7 +217,6 @@ def create_scored_boundary_layer(self, clip_area: QgsGeometry, index: int) -> Qg ) layer = QgsVectorLayer(shapefile_path, "area_layer", "ogr") self.progressChanged.emit(80.0) # We just use nominal intervals for progress updates - return layer # Default implementation of the abstract method - not used in this workflow @@ -167,16 +227,15 @@ def _process_raster_for_area( current_bbox: QgsGeometry, area_raster: str, index: int, + area_name: str = None, ): """ Executes the actual workflow logic for a single area using a raster. - :current_area: Current polygon from our study area. :clip_area: Polygon to clip the raster to which is aligned to cell edges. :current_bbox: Bounding box of the above area. :area_raster: A raster layer of features to analyse that includes only bbox pixels in the study area. :index: Index of the current area. - :return: Path to the reclassified raster. """ pass @@ -187,6 +246,7 @@ def _process_aggregate_for_area( clip_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: str = None, ): """ Executes the workflow, reporting progress through the feedback object and checking for cancellation. diff --git a/geest/core/workflows/dont_use_workflow.py b/geest/core/workflows/dont_use_workflow.py index f7f13dad..ae030226 100644 --- a/geest/core/workflows/dont_use_workflow.py +++ b/geest/core/workflows/dont_use_workflow.py @@ -53,6 +53,7 @@ def _process_raster_for_area( current_bbox: QgsGeometry, area_raster: str, index: int, + area_name: str = None, ): """ Executes the actual workflow logic for a single area using a raster. @@ -73,6 +74,7 @@ def _process_aggregate_for_area( clip_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: str = None, ): """ Executes the workflow, reporting progress through the feedback object and checking for cancellation. diff --git a/geest/core/workflows/eplex_workflow.py b/geest/core/workflows/eplex_workflow.py index ddb4e071..9b0a422d 100644 --- a/geest/core/workflows/eplex_workflow.py +++ b/geest/core/workflows/eplex_workflow.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- """📦 EPLEX Workflow module. - This module contains functionality for EPLEX score workflow. """ - import os from qgis.core import ( @@ -27,7 +25,6 @@ class EPLEXWorkflow(WorkflowBase): """ Concrete implementation of 'use_eplex_score' workflow. - Creates a raster filled with the EPLEX score value for the study area. This is used when women considerations are disabled, providing a single contextual score based on Employment Protection Legislation Index. @@ -43,7 +40,6 @@ def __init__( working_directory: str = None, ): """Initialize the EPLEX workflow with attributes and feedback. - Args: item: JsonTreeItem representing the indicator to process. cell_size_m: Cell size in meters for rasterization. @@ -67,7 +63,6 @@ def __init__( tag="GeoE3", level=Qgis.Info, ) - self.features_layer = True # Not needed for this workflow self.workflow_name = "eplex_score" @@ -109,44 +104,34 @@ def _process_features_for_area( current_bbox: QgsGeometry, area_features, index: int, + area_name: str = None, ) -> str: """Create a raster filled with EPLEX score for the study area. - Uses the grid layer directly and rasterizes it with the EPLEX score value. - Args: current_area: Current polygon from our study area. clip_area: Polygon to clip the raster to, aligned to cell edges. current_bbox: Bounding box of the above area. area_features: Not used in this workflow. index: Iteration / number of area being processed. - Returns: Raster file path of the output. """ log_message(f"Processing area {index} for EPLEX score workflow", tag="GeoE3", level=Qgis.Info) - self.progressChanged.emit(10.0) - # Create a memory layer with a single feature covering the clip area fields = QgsFields() fields.append(QgsField("value", QVariant.Double)) - eplex_layer = QgsVectorLayer(f"Polygon?crs={self.target_crs.authid()}", "eplex_temp", "memory") eplex_layer.dataProvider().addAttributes(fields) eplex_layer.updateFields() - self.progressChanged.emit(30.0) - # Create a single feature with the clip_area geometry and EPLEX score feature = QgsFeature(fields) feature.setGeometry(clip_area) feature.setAttribute("value", self.eplex_score) - eplex_layer.dataProvider().addFeatures([feature]) - self.progressChanged.emit(50.0) - # Rasterize this layer output_path = self._rasterize( eplex_layer, @@ -155,9 +140,7 @@ def _process_features_for_area( value_field="value", default_value=0, ) - self.progressChanged.emit(90.0) - if output_path and os.path.exists(output_path): log_message( f"EPLEX raster created successfully: {output_path}", @@ -171,9 +154,7 @@ def _process_features_for_area( level=Qgis.Critical, ) return None - self.progressChanged.emit(100.0) - return output_path # Default implementations of abstract methods - not used in this workflow @@ -184,9 +165,9 @@ def _process_raster_for_area( current_bbox: QgsGeometry, area_raster: str, index: int, + area_name: str = None, ): """Not used in EPLEX workflow. - Args: current_area: Current polygon from study area. clip_area: Polygon to clip the raster to. @@ -202,9 +183,9 @@ def _process_aggregate_for_area( clip_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: str = None, ): """Not used in EPLEX workflow. - Args: current_area: Current polygon from study area. clip_area: Polygon to clip the raster to. diff --git a/geest/core/workflows/index_score_with_ghsl_workflow.py b/geest/core/workflows/index_score_with_ghsl_workflow.py index a9be8c14..320e2552 100644 --- a/geest/core/workflows/index_score_with_ghsl_workflow.py +++ b/geest/core/workflows/index_score_with_ghsl_workflow.py @@ -1,27 +1,25 @@ # -*- coding: utf-8 -*- -"""📦 Index Score With Ghsl Workflow module. - +"""📦 Index Score With GHSL Workflow module. This module contains functionality for index score with ghsl workflow. """ import os from typing import Optional -from qgis import processing # noqa: F401 # QGIS processing toolbox -from qgis.core import ( # noqa: F401 - QgsFeature, - QgsFeatureRequest, +from qgis.core import ( + Qgis, QgsFeedback, - QgsField, QgsGeometry, QgsProcessingContext, - QgsVectorDataProvider, - QgsVectorFileWriter, QgsVectorLayer, ) -from qgis.PyQt.QtCore import QVariant from geest.core import JsonTreeItem +from geest.core.grid_column_utils import ( + clear_grid_column, + rasterize_grid_column, + write_spatial_join_to_grid, +) from geest.utilities import log_message from .workflow_base import WorkflowBase @@ -36,11 +34,9 @@ class IndexScoreWithGHSLException(Exception): class IndexScoreWithGHSLWorkflow(WorkflowBase): """ Concrete implementation of a 'use_index_score_with_ghsl' workflow. - This workflow scores areas using an index value, masked to GHSL settlement boundaries. - Study area clip polygons are pre-filtered during study area creation to only include - areas that intersect GHSL, so this workflow intersects with GHSL to get the precise - settlement boundaries for scoring. + Grid cells that intersect GHSL settlements get the index score; others stay NULL. + Uses grid-first approach: spatial join directly to grid, then rasterize for VRT output. """ def __init__( @@ -54,35 +50,30 @@ def __init__( ): """ Initialize the workflow with attributes and feedback. - Args: item: JsonTreeItem representing the analysis, dimension, or factor to process. cell_size_m: Cell size in meters for rasterization. analysis_scale: Scale of the analysis, e.g., 'local', 'national' feedback: QgsFeedback object for progress reporting and cancellation. - context: QgsProcessingContext object for processing. This can be used to pass objects to the thread. e.g. the QgsProject Instance - working_directory: Folder containing study_area.gpkg and where the outputs will be placed. If not set will be taken from QSettings. + context: QgsProcessingContext object for processing. + working_directory: Folder containing study_area.gpkg and where the outputs will be placed. """ log_message("\n\n\n\n") log_message("--------------------------------------------") log_message("Initializing Index Score with GHSL Workflow") log_message("--------------------------------------------") - super().__init__( - item, cell_size_m, analysis_scale, feedback, context, working_directory - ) # ⭐️ Item is a reference - whatever you change in this item will directly update the tree + super().__init__(item, cell_size_m, analysis_scale, feedback, context, working_directory) index_score = self.attributes.get("index_score", 0) log_message(f"Index score before rescaling to likert scale: {index_score}") self.index_score = (float(index_score) / 100) * 5 log_message(f"Index score after rescaling to likert scale: {self.index_score}") - self.features_layer = ( - True # Normally we would set this to a QgsVectorLayer but in this workflow it is not needed - ) + self.features_layer = True self.workflow_name = "index_score" + self.use_grid_first = True + self._column_cleared = False # Get the analysis extents self.study_area_bbox = self._study_area_bbox_4326() - self.ghsl_layer_path = f"{self.gpkg_path}|layername=ghsl_settlements" - # Check if GHSL layer exists, try to download if not if not self.ensure_ghsl_data(): log_message( @@ -90,7 +81,6 @@ def __init__( level="WARNING", ) else: - # Verify the layer is valid after ensuring data exists ghsl_layer = QgsVectorLayer(self.ghsl_layer_path, "ghsl_layer", "ogr") if not ghsl_layer.isValid(): log_message( @@ -105,117 +95,72 @@ def _process_features_for_area( current_bbox: QgsGeometry, area_features: QgsVectorLayer, index: int, + area_name: str = None, ) -> str: """ - Executes the actual workflow logic for a single area - Must be implemented by sub classes. - + Score grid cells that intersect GHSL settlements, then rasterize from grid. Args: current_area: Current polygon from our study area. clip_area: Current area but expanded to coincide with grid cell boundaries. current_bbox: Bounding box of the above area. - area_features: A vector layer of features to analyse that includes only features in the study area. + area_features: A vector layer of features to analyse (unused). index: Iteration / number of area being processed. - + area_name: Name of the current area. Returns: Raster file path of the output. """ _ = area_features # unused + _ = current_area # unused log_message(f"Processing area {index} with index score {self.index_score}") self.progressChanged.emit(10.0) - # Load GHSL layer and get features intersecting this area - # Clip polygons are pre-filtered during study area creation, so we just need - # to intersect with GHSL to get precise settlement boundaries for scoring - ghsl_layer = QgsVectorLayer(self.ghsl_layer_path, "ghsl_layer", "ogr") - if not ghsl_layer.isValid(): - log_message(f"GHSL layer not valid, using full clip area for area {index}") - masked_geom = clip_area - else: - # Use QgsFeatureRequest spatial filter for cross-platform reliability - request = QgsFeatureRequest().setFilterRect(current_area.boundingBox()) - ghsl_geometries = [] - for feat in ghsl_layer.getFeatures(request): - if feat.geometry().intersects(current_area): - ghsl_geometries.append(feat.geometry()) - - if ghsl_geometries: - ghsl_union = QgsGeometry.unaryUnion(ghsl_geometries) - masked_geom = clip_area.intersection(ghsl_union) - if masked_geom.isEmpty(): - log_message(f"GHSL intersection empty for area {index}, using full clip area") - masked_geom = clip_area - else: - log_message(f"No GHSL features found for area {index}, using full clip area") - masked_geom = clip_area - - self.progressChanged.emit(40.0) - - # Create scored layer with GHSL-masked geometry - scored_layer = self.create_scored_boundary_layer(clip_area=masked_geom, index=index) - self.progressChanged.emit(60.0) - - # Rasterize - raster_output = self._rasterize( - scored_layer, - current_bbox, - index, - value_field="score", - default_value=0, + # Clear grid column once at start + if not self._column_cleared: + clear_grid_column(self.gpkg_path, self.layer_id) + self._column_cleared = True + + # Spatial join: set index_score for grid cells that intersect GHSL settlements + score = self.index_score + updated = write_spatial_join_to_grid( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + features_gpkg=self.gpkg_path, + features_layer="ghsl_settlements", + score_expression=lambda feat: score, + area_name=area_name, + aggregation_method="MAX", + save_buffers=False, ) - self.progressChanged.emit(100.0) - - log_message(f"Raster output: {raster_output}") - return raster_output - - def create_scored_boundary_layer(self, clip_area: QgsGeometry, index: int) -> QgsVectorLayer: - """ - Create a scored boundary layer, filtering features by the current_area. - - Args: - clip_area: The clipping area geometry. - index: The index of the current processing area. - - Returns: - A vector layer with a 'score' attribute. - """ - output_prefix = f"{self.layer_id}_area_{index}" - - self.progressChanged.emit(20.0) # We just use nominal intervals for progress updates - # Create a new memory layer with the target CRS (EPSG:4326) - subset_layer = QgsVectorLayer("Polygon", "subset", "memory") - subset_layer.setCrs(self.target_crs) - subset_layer_data: QgsVectorDataProvider = subset_layer.dataProvider() - field = QgsField("score", QVariant.Double) - fields = [field] - # Add attributes (fields) from the point_layer - subset_layer_data.addAttributes(fields) - subset_layer.updateFields() - self.progressChanged.emit(40.0) # We just use nominal intervals for progress updates + self.progressChanged.emit(60.0) - feature = QgsFeature(subset_layer.fields()) - feature.setGeometry(clip_area) - score_field_index = subset_layer.fields().indexFromName("score") - feature.setAttribute(score_field_index, self.index_score) - features = [feature] - # Add reprojected features to the new subset layer - subset_layer_data.addFeatures(features) - subset_layer.commitChanges() - self.progressChanged.emit(60.0) # We just use nominal intervals for progress updates + if updated >= 0: + log_message(f"Updated {updated} grid cells with GHSL-masked index score for area {area_name}") + else: + log_message( + f"Failed to write GHSL-masked index score for area {area_name}", + tag="GeoE3", + level=Qgis.Warning, + ) - shapefile_path = os.path.join(self.workflow_directory, f"{output_prefix}.shp") - # Use QgsVectorFileWriter to save the layer to a shapefile - QgsVectorFileWriter.writeAsVectorFormat( - subset_layer, - shapefile_path, - "utf-8", - subset_layer.crs(), - "ESRI Shapefile", + # Rasterize from grid column for VRT output + output_path = os.path.join( + self.workflow_directory, + f"{self.layer_id}_ghsl_scored_{index}.tif", ) - layer = QgsVectorLayer(shapefile_path, "area_layer", "ogr") - self.progressChanged.emit(80.0) # We just use nominal intervals for progress updates - - return layer + rect = current_bbox.boundingBox() + extent = (rect.xMinimum(), rect.yMinimum(), rect.xMaximum(), rect.yMaximum()) + rasterize_grid_column( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + output_raster_path=output_path, + cell_size=self.cell_size_m, + extent=extent, + nodata=-9999.0, + area_name=area_name, + ) + self.progressChanged.emit(100.0) + log_message(f"Rasterized grid column to {output_path}") + return output_path # Default implementation of the abstract method - not used in this workflow def _process_raster_for_area( @@ -225,20 +170,9 @@ def _process_raster_for_area( current_bbox: QgsGeometry, area_raster: str, index: int, + area_name: str = None, ): - """ - Executes the actual workflow logic for a single area using a raster. - - Args: - current_area: Current polygon from our study area. - clip_area: Polygon to clip the raster to which is aligned to cell edges. - current_bbox: Bounding box of the above area. - area_raster: A raster layer of features to analyse that includes only bbox pixels in the study area. - index: Index of the current area. - - Returns: - Path to the reclassified raster. - """ + """Not used in this workflow.""" return None def _process_aggregate_for_area( @@ -247,17 +181,7 @@ def _process_aggregate_for_area( clip_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: str = None, ): - """ - Executes the actual workflow logic for a single area using an aggregate. - - Args: - current_area: Current polygon from our study area. - clip_area: Polygon to clip the raster to which is aligned to cell edges. - current_bbox: Bounding box of the above area. - index: Index of the current area. - - Returns: - Path to the reclassified raster. - """ + """Not used in this workflow.""" return None diff --git a/geest/core/workflows/index_score_with_ookla_workflow.py b/geest/core/workflows/index_score_with_ookla_workflow.py index 699dc193..27c765b2 100644 --- a/geest/core/workflows/index_score_with_ookla_workflow.py +++ b/geest/core/workflows/index_score_with_ookla_workflow.py @@ -1,29 +1,25 @@ # -*- coding: utf-8 -*- """📦 Index Score With Ookla Workflow module. - This module contains functionality for index score with ookla workflow. """ - import os from typing import Optional -from qgis import processing # noqa: F401 # QGIS processing toolbox -from qgis.core import ( # noqa: F401 +from qgis.core import ( Qgis, - QgsDataProvider, - QgsFeature, QgsFeedback, - QgsField, QgsGeometry, QgsProcessingContext, - QgsVectorDataProvider, - QgsVectorFileWriter, QgsVectorLayer, ) -from qgis.PyQt.QtCore import QVariant from geest.core import JsonTreeItem from geest.core.algorithms.ookla_downloader import OoklaDownloader +from geest.core.grid_column_utils import ( + clear_grid_column, + rasterize_grid_column, + write_spatial_join_to_grid, +) from geest.utilities import log_message from .workflow_base import WorkflowBase @@ -45,7 +41,6 @@ def setProgress(self, progress): super().setProgress(progress) if self.base_feedback: self.base_feedback.setProgress(progress) - # Emit to workflow progress bar self.workflow.progressChanged.emit(float(progress)) def isCanceled(self): @@ -58,10 +53,9 @@ def isCanceled(self): class IndexScoreWithOoklaWorkflow(WorkflowBase): """ Concrete implementation of a 'use_index_score_with_ookla' workflow. - - This follows the same logic as the index score workflow but additionally - masks the result using the Ookla coverage layer to ensure that only areas - that have Ookla data are included in the final output. + This workflow scores areas using an index value, masked to Ookla broadband coverage. + Grid cells that intersect Ookla coverage tiles get the index score; others stay NULL. + Uses grid-first approach: spatial join directly to grid, then rasterize for VRT output. """ def __init__( @@ -79,27 +73,24 @@ def __init__( :param cell_size_m: Cell size in meters for rasterization. :param analysis_scale: Scale of the analysis, e.g., 'local', 'national' :param feedback: QgsFeedback object for progress reporting and cancellation. - :param context: QgsProcessingContext object for processing. This can be used to pass objects to the thread. e.g. the QgsProject Instance - :param working_directory: Folder containing study_area.gpkg and where the outputs will be placed. If not set will be taken from QSettings. + :param context: QgsProcessingContext object for processing. + :param working_directory: Folder containing study_area.gpkg and outputs. """ log_message("\n\n\n\n") log_message("--------------------------------------------") log_message("Initializing Index Score with Ookla Workflow") log_message("--------------------------------------------") - super().__init__( - item, cell_size_m, analysis_scale, feedback, context, working_directory - ) # ⭐️ Item is a reference - whatever you change in this item will directly update the tree + super().__init__(item, cell_size_m, analysis_scale, feedback, context, working_directory) index_score = self.attributes.get("index_score", 0) log_message(f"Index score before rescaling to likert scale: {index_score}") self.index_score = (float(index_score) / 100) * 5 log_message(f"Index score after rescaling to likert scale: {self.index_score}") - self.features_layer = ( - True # Normally we would set this to a QgsVectorLayer but in this workflow it is not needed - ) + self.features_layer = True self.workflow_name = "index_score" + self.use_grid_first = True + self._column_cleared = False # Get the analysis extents self.study_area_bbox = self._study_area_bbox_4326() - # Lazy load OOKLA data during execute to avoid blocking __init__ self.ookla_layer_path = None self.ookla_downloaded = False @@ -111,16 +102,10 @@ def _download_ookla_data(self): """ if self.ookla_downloaded: return - log_message("Downloading Ookla data (this may take several minutes)...") self.updateStatus("Downloading Ookla data — this may take several minutes...") self.progressChanged.emit(1.0) - - # Bridge feedback to workflow progress signals bridge_feedback = ProgressBridgeFeedback(self, self.feedback) - - # Prepare Ookla coverage layer - adds a minute or two to the workflow - # and requires internet access ookla_layer_path = os.path.join(self.working_directory, "study_area") log_message(f"Ookla output will be saved to: {ookla_layer_path}") downloader = OoklaDownloader( @@ -129,7 +114,7 @@ def _download_ookla_data(self): filename_prefix="ookla", use_cache=True, delete_existing=True, - feedback=bridge_feedback, # Use bridge feedback for progress visibility + feedback=bridge_feedback, ) self.updateStatus("Ookla: fetching broadband data (may take several minutes)...") try: @@ -151,121 +136,73 @@ def _process_features_for_area( current_bbox: QgsGeometry, area_features: QgsVectorLayer, index: int, + area_name: str = None, ) -> str: """ - Executes the actual workflow logic for a single area - Must be implemented by sub classes. - + Score grid cells that intersect Ookla coverage, then rasterize from grid. :current_area: Current polygon from our study area. :current_bbox: Bounding box of the above area. - :area_features: A vector layer of features to analyse that includes only features in the study area. + :area_features: A vector layer of features to analyse (unused). :index: Iteration / number of area being processed. - + :area_name: Name of the current area. :return: Raster file path of the output. """ _ = area_features # unused + _ = current_area # unused # Download OOKLA data on first area if index == 0: self._download_ookla_data() - log_message(f"Index score: {self.index_score}") - self.progressChanged.emit(10.0) # We just use nominal intervals for progress updates + self.progressChanged.emit(10.0) + + # Clear grid column once at start + if not self._column_cleared: + clear_grid_column(self.gpkg_path, self.layer_id) + self._column_cleared = True + + # Spatial join: set index_score for grid cells that intersect Ookla coverage + score = self.index_score + updated = write_spatial_join_to_grid( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + features_gpkg=self.ookla_layer_path, + features_layer="ookla_combined", + score_expression=lambda feat: score, + area_name=area_name, + aggregation_method="MAX", + save_buffers=False, + ) + self.progressChanged.emit(60.0) - # Mask with OOKLA coverage - ookla_layer = QgsVectorLayer(self.ookla_layer_path, "ookla_layer", "ogr") - expr = f"intersects($geometry, geom_from_wkt('{current_area.asWkt()}'))" - ookla_layer.selectByExpression(expr, QgsVectorLayer.SetSelection) - ookla_features = ookla_layer.selectedFeatures() - final_geom: QgsGeometry = None - if ookla_features: - ookla_union_geom = QgsGeometry.unaryUnion([feat.geometry() for feat in ookla_features]) - final_geom = clip_area.intersection(ookla_union_geom) + if updated >= 0: + log_message(f"Updated {updated} grid cells with Ookla-masked index score for area {area_name}") else: - log_message(f"No Ookla coverage in area {index}, skipping ookla masking.") - - if not final_geom or final_geom.isEmpty(): - log_message(f"No Ookla coverage in area {index}, using full clip area with score 0.") - final_geom = clip_area - - # Create scored layer only if we have valid geometry - scored_layer = self.create_scored_boundary_layer( - clip_area=final_geom, - index=index, + log_message( + f"Failed to write Ookla-masked index score for area {area_name}", + tag="GeoE3", + level=Qgis.Warning, + ) + + # Rasterize from grid column for VRT output + output_path = os.path.join( + self.workflow_directory, + f"{self.layer_id}_ookla_scored_{index}.tif", ) - self.progressChanged.emit(60.0) # We just use nominal intervals for progress - - # Rasterize - raster_output = self._rasterize( - scored_layer, - current_bbox, - index, - value_field="score", - default_value=0, + rect = current_bbox.boundingBox() + extent = (rect.xMinimum(), rect.yMinimum(), rect.xMaximum(), rect.yMaximum()) + rasterize_grid_column( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + output_raster_path=output_path, + cell_size=self.cell_size_m, + extent=extent, + nodata=-9999.0, + area_name=area_name, ) - self.progressChanged.emit(100.0) # We just use nominal intervals for progress updates - - log_message(f"Raster output: {raster_output}") - log_message(f"Workflow completed for area {index}") - return raster_output - - def create_scored_boundary_layer(self, clip_area: QgsGeometry, index: int) -> QgsVectorLayer: - """ - Create a scored boundary layer, filtering features by the current_area. - - :param index: The index of the current processing area. - :return: A vector layer with a 'score' attribute. - """ - output_prefix = f"{self.layer_id}_area_{index}" - - self.progressChanged.emit(20.0) # We just use nominal intervals for progress updates - # Create memory layer - subset_layer = QgsVectorLayer("Polygon", "subset", "memory") - subset_layer.setCrs(self.target_crs) - subset_layer_data: QgsVectorDataProvider = subset_layer.dataProvider() - field = QgsField("score", QVariant.Double) - fields = [field] - subset_layer_data.addAttributes(fields) - subset_layer.updateFields() - self.progressChanged.emit(40.0) # We just use nominal intervals for progress updates - - feature = QgsFeature(subset_layer.fields()) - feature.setGeometry(clip_area) - score_field_index = subset_layer.fields().indexFromName("score") - feature.setAttribute(score_field_index, self.index_score) - features = [feature] - subset_layer_data.addFeatures(features) - subset_layer.commitChanges() - self.progressChanged.emit(60.0) # We just use nominal intervals for progress updates - - shapefile_path = os.path.join(self.workflow_directory, f"{output_prefix}.shp") - os.makedirs(self.workflow_directory, exist_ok=True) - - # Write to shapefile - error, error_string = QgsVectorFileWriter.writeAsVectorFormat( - subset_layer, - shapefile_path, - "utf-8", - subset_layer.crs(), - "ESRI Shapefile", - ) - - if error != QgsVectorFileWriter.NoError: - log_message(f"Error writing shapefile: {error_string} (code: {error})") - return None - - if not os.path.exists(shapefile_path): - log_message(f"Error: Shapefile not created at {shapefile_path}") - return None - - layer = QgsVectorLayer(shapefile_path, "area_layer", "ogr") - - if not layer.isValid(): - log_message(f"Error loading layer: {layer.error().message()}") - return None - self.progressChanged.emit(80.0) # We just use nominal intervals for progress updates - - return layer + self.progressChanged.emit(100.0) + log_message(f"Rasterized grid column to {output_path}") + return output_path # Default implementation of the abstract method - not used in this workflow def _process_raster_for_area( @@ -275,18 +212,9 @@ def _process_raster_for_area( current_bbox: QgsGeometry, area_raster: str, index: int, + area_name: str = None, ): - """ - Executes the actual workflow logic for a single area using a raster. - - :current_area: Current polygon from our study area. - :clip_area: Polygon to clip the raster to which is aligned to cell edges. - :current_bbox: Bounding box of the above area. - :area_raster: A raster layer of features to analyse that includes only bbox pixels in the study area. - :index: Index of the current area. - - :return: Path to the reclassified raster. - """ + """Not used in this workflow.""" pass def _process_aggregate_for_area( @@ -295,8 +223,7 @@ def _process_aggregate_for_area( clip_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: str = None, ): - """ - Executes the workflow, reporting progress through the feedback object and checking for cancellation. - """ + """Not used in this workflow.""" pass diff --git a/geest/core/workflows/index_score_workflow.py b/geest/core/workflows/index_score_workflow.py index 89936ed2..d7e3f5e6 100644 --- a/geest/core/workflows/index_score_workflow.py +++ b/geest/core/workflows/index_score_workflow.py @@ -2,9 +2,13 @@ """📦 Index Score Workflow module. This module contains functionality for index score workflow. + +Supports grid-first mode where the index score is written directly to +the study_area_grid column, then optionally rasterized. """ import os +from typing import Optional from qgis import processing # noqa: F401 # QGIS processing toolbox from qgis.core import ( # noqa: F401 @@ -20,6 +24,10 @@ from qgis.PyQt.QtCore import QVariant from geest.core import JsonTreeItem +from geest.core.grid_column_utils import ( + rasterize_grid_column, + write_uniform_value_to_grid, +) from geest.utilities import log_message from .workflow_base import WorkflowBase @@ -61,6 +69,8 @@ def __init__( True # Normally we would set this to a QgsVectorLayer but in this workflow it is not needed ) self.workflow_name = "index_score" + # Grid-first mode: write results to grid columns first, then rasterize + self.use_grid_first = True def _process_features_for_area( self, @@ -69,31 +79,61 @@ def _process_features_for_area( current_bbox: QgsGeometry, area_features: QgsVectorLayer, index: int, + area_name: Optional[str] = None, ) -> str: """ - Executes the actual workflow logic for a single area - Must be implemented by subclasses. + Executes the actual workflow logic for a single area. - :current_area: Current polygon from our study area. - :current_bbox: Bounding box of the above area. - :area_features: A vector layer of features to analyse that includes only features in the study area. - :index: Iteration / number of area being processed. + Supports both raster-first (legacy) and grid-first modes. - :return: Raster file path of the output. + Args: + current_area: Current polygon from our study area. + clip_area: Clipping polygon aligned to grid cells. + current_bbox: Bounding box of the above area. + area_features: A vector layer of features to analyse (unused for index_score). + index: Iteration / number of area being processed. + area_name: Name of the area being processed (for grid-first mode). + + Returns: + Raster file path of the output. """ _ = area_features # unused - log_message(f"Processing area {index} score workflow") + log_message(f"Processing area {index} score workflow (grid_first={self.use_grid_first})") log_message(f"Index score: {self.index_score}") - self.progressChanged.emit(10.0) # We just use nominal intervals for progress updates + self.progressChanged.emit(10.0) + + if self.use_grid_first and area_name: + return self._process_grid_first( + current_bbox=current_bbox, + index=index, + area_name=area_name, + ) + else: + return self._process_raster_first( + clip_area=clip_area, + current_bbox=current_bbox, + index=index, + ) + + def _process_raster_first( + self, + clip_area: QgsGeometry, + current_bbox: QgsGeometry, + index: int, + ) -> str: + """Legacy raster-first processing. + Creates a polygon layer with the score and rasterizes it. + """ # Create a scored boundary layer filtered by current_area scored_layer = self.create_scored_boundary_layer( clip_area=clip_area, index=index, ) - self.progressChanged.emit(30.0) # We just use nominal intervals for progress updates - # Create a scored boundary layer + self.progressChanged.emit(30.0) + + # Rasterize the scored layer raster_output = self._rasterize( scored_layer, current_bbox, @@ -101,12 +141,73 @@ def _process_features_for_area( value_field="score", default_value=255, ) - self.progressChanged.emit(100.0) # We just use nominal intervals for progress updates + self.progressChanged.emit(100.0) log_message(f"Raster output: {raster_output}") log_message(f"Workflow completed for area {index}") return raster_output + def _process_grid_first( + self, + current_bbox: QgsGeometry, + index: int, + area_name: str, + ) -> str: + """Grid-first processing. + + Writes the index score directly to the grid column, then rasterizes. + """ + # Step 1: Write uniform value to grid + self.progressChanged.emit(20.0) + log_message(f"Writing index score {self.index_score} to grid column {self.layer_id} for area {area_name}") + + updated_count = write_uniform_value_to_grid( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + value=self.index_score, + area_name=area_name, + ) + + if updated_count < 0: + log_message(f"Failed to write index score to grid for area {area_name}", level=Qgis.Warning) + # Fall back to raster-first method + return self._process_raster_first( + clip_area=None, # Not available in this path + current_bbox=current_bbox, + index=index, + ) + + log_message(f"Updated {updated_count} grid cells with index score {self.index_score}") + self.progressChanged.emit(50.0) + + # Step 2: Rasterize from grid column + output_path = os.path.join( + self.workflow_directory, + f"{self.layer_id}_{index}.tif", + ) + + rect = current_bbox.boundingBox() + extent = (rect.xMinimum(), rect.yMinimum(), rect.xMaximum(), rect.yMaximum()) + + success = rasterize_grid_column( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + output_raster_path=output_path, + cell_size=self.cell_size_m, + extent=extent, + nodata=-9999.0, + area_name=area_name, + ) + + self.progressChanged.emit(100.0) + + if success: + log_message(f"Rasterized grid column to {output_path}") + return output_path + else: + log_message(f"Failed to rasterize grid column for area {area_name}", level=Qgis.Warning) + return None + def create_scored_boundary_layer(self, clip_area: QgsGeometry, index: int) -> QgsVectorLayer: """ Create a scored boundary layer, filtering features by the current_area. @@ -180,6 +281,7 @@ def _process_aggregate_for_area( clip_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: str = None, ): """ Executes the workflow, reporting progress through the feedback object and checking for cancellation. diff --git a/geest/core/workflows/multi_buffer_distances_native_workflow.py b/geest/core/workflows/multi_buffer_distances_native_workflow.py index dac150d4..a2fee6a0 100644 --- a/geest/core/workflows/multi_buffer_distances_native_workflow.py +++ b/geest/core/workflows/multi_buffer_distances_native_workflow.py @@ -1,6 +1,12 @@ # -*- coding: utf-8 -*- -"""Multi-buffer distances workflow using native QGIS network analysis.""" +"""Multi-buffer distances workflow using native QGIS network analysis. + +Supports grid-first mode where buffer scores are written directly to +the study_area_grid column, then rasterized. +""" + import os +from typing import Optional from urllib.parse import unquote from qgis import processing @@ -20,6 +26,11 @@ from geest.core import JsonTreeItem from geest.core.algorithms import NativeNetworkAnalysisProcessingTask +from geest.core.grid_column_utils import ( + clear_grid_column, + rasterize_grid_column, + write_buffer_values_to_grid, +) from geest.core.workflows.mappings import MAPPING_REGISTRY from geest.utilities import log_message @@ -30,7 +41,7 @@ class MultiBufferDistancesNativeWorkflow(WorkflowBase): """Multi-buffer workflow using native QGIS network analysis. Creates concentric isochrones around points using road network distances. - Results are rasterized and combined into a VRT. + Results are written to grid columns and rasterized. """ def __init__( @@ -108,7 +119,6 @@ def __init__( level=Qgis.Warning, ) raise Exception("Invalid travel distances provided.") - layer_path = self.attributes.get("multi_buffer_point_shapefile", None) if layer_path: layer_path = unquote(layer_path) @@ -135,7 +145,6 @@ def __init__( level=Qgis.Warning, ) raise Exception("Invalid points layer found.") - mode = self.attributes.get("multi_buffer_travel_mode", "Walking") self.mode = None if mode == "Walking": @@ -154,14 +163,20 @@ def __init__( ) raise Exception("Invalid network layer found.") log_message("Multi Buffer Distances Native Workflow initialized") + self.workflow_name = "multi_buffer_point" + # Grid-first mode: write results to grid columns first, then rasterize + self.use_grid_first = True + # Track if we've cleared the column (only do once, not per area) + self._column_cleared = False def _process_features_for_area( self, - current_area: "QgsGeometry", - clip_area: "QgsGeometry", - current_bbox: "QgsGeometry", - area_features: "QgsVectorLayer", + current_area: QgsGeometry, + clip_area: QgsGeometry, + current_bbox: QgsGeometry, + area_features: QgsVectorLayer, index: int, + area_name: Optional[str] = None, ) -> str: """Process a single area. @@ -171,30 +186,56 @@ def _process_features_for_area( current_bbox: Bounding box. area_features: Features to analyze. index: Area number being processed. + area_name: Name of the area being processed. Returns: Raster file path, or False if failed. """ + if self.use_grid_first: + return self._process_grid_first( + current_area=current_area, + clip_area=clip_area, + current_bbox=current_bbox, + area_features=area_features, + index=index, + area_name=area_name, + ) + else: + return self._process_raster_first( + current_area=current_area, + clip_area=clip_area, + current_bbox=current_bbox, + area_features=area_features, + index=index, + ) + + def _process_raster_first( + self, + current_area: QgsGeometry, + clip_area: QgsGeometry, + current_bbox: QgsGeometry, + area_features: QgsVectorLayer, + index: int, + ) -> str: + """Legacy raster-first processing.""" # Check if we should use simple buffer (Regional scale) instead of network analysis if self.use_simple_buffer and self.buffer_distance: log_message( f"Using simple buffer for Regional scale: {self.buffer_distance}m", level=Qgis.Info, ) - return self._process_features_with_simple_buffer( + return self._process_features_with_simple_buffer_legacy( current_area=current_area, clip_area=clip_area, current_bbox=current_bbox, area_features=area_features, index=index, ) - # Original network analysis approach for National/Local scale log_message( f"Starting network analysis for area {index + 1}", level=Qgis.Info, ) - isochrones_gpkg = self.create_isochrones( point_layer=area_features, clip_geometry=current_area, @@ -207,23 +248,165 @@ def _process_features_for_area( level=Qgis.Warning, ) return False - bands = self._create_bands(isochrones_gpkg_path=isochrones_gpkg, index=index) scored_buffers = self._assign_scores(bands) - if scored_buffers is False: log_message("No scored buffers were created.", level=Qgis.Warning) return False - raster_output = self._rasterize( input_layer=scored_buffers, bbox=current_bbox, index=index, value_field="value", ) - return raster_output + def _process_grid_first( + self, + current_area: QgsGeometry, + clip_area: QgsGeometry, + current_bbox: QgsGeometry, + area_features: QgsVectorLayer, + index: int, + area_name: str, + ) -> str: + """Grid-first processing - writes buffer scores directly to study_area_grid.""" + # Clear column once at the start (not per area) + if not self._column_cleared: + log_message(f"Clearing column {self.layer_id} before processing") + clear_grid_column(self.gpkg_path, self.layer_id) + self._column_cleared = True + + self.progressChanged.emit(5.0) + + # Check if we should use simple buffer (Regional scale) instead of network analysis + if self.use_simple_buffer and self.buffer_distance: + log_message( + f"Using simple buffer for Regional scale: {self.buffer_distance}m", + level=Qgis.Info, + ) + scored_buffers = self._create_simple_buffer_scored(area_features, index) + else: + # Original network analysis approach for National/Local scale + log_message( + f"Starting network analysis for area {index + 1}", + level=Qgis.Info, + ) + isochrones_gpkg = self.create_isochrones( + point_layer=area_features, + clip_geometry=current_area, + area_index=index, + ) + if not isochrones_gpkg: + log_message( + f"No isochrones created for area {index}.", + tag="GeoE3", + level=Qgis.Warning, + ) + return False + + self.progressChanged.emit(40.0) + + bands = self._create_bands(isochrones_gpkg_path=isochrones_gpkg, index=index) + scored_buffers = self._assign_scores(bands) + + if scored_buffers is False: + log_message("No scored buffers were created.", level=Qgis.Warning) + return False + + self.progressChanged.emit(50.0) + + # Write buffer scores to grid cells + log_message(f"Writing buffer scores to grid column {self.layer_id}") + write_buffer_values_to_grid( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + buffer_layer=scored_buffers, + value_field="value", + aggregation_method="MAX", + feedback=self.feedback, + ) + + self.progressChanged.emit(80.0) + + # Rasterize from grid column + output_path = os.path.join( + self.workflow_directory, + f"{self.layer_id}_{index}.tif", + ) + + rect = current_bbox.boundingBox() + extent = (rect.xMinimum(), rect.yMinimum(), rect.xMaximum(), rect.yMaximum()) + + rasterize_grid_column( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + output_raster_path=output_path, + cell_size=self.cell_size_m, + extent=extent, + nodata=-9999.0, + area_name=area_name, + ) + + self.progressChanged.emit(100.0) + log_message(f"Rasterized grid column to {output_path}") + return output_path + + def _create_simple_buffer_scored( + self, + area_features: QgsVectorLayer, + index: int, + ) -> QgsVectorLayer: + """Create simple buffer and assign scores for Regional scale. + + Args: + area_features: Features to buffer. + index: Area index. + + Returns: + Scored buffer layer with "value" field. + """ + buffer_output = os.path.join(self.workflow_directory, f"simple_buffer_{index}.gpkg") + if os.path.exists(buffer_output): + os.remove(buffer_output) + + buffer_params = { + "INPUT": area_features, + "DISTANCE": self.buffer_distance, + "OUTPUT": buffer_output, + } + result = processing.run("native:buffer", buffer_params, feedback=QgsProcessingFeedback()) + buffered_layer_path = result["OUTPUT"] + + if not buffered_layer_path or not os.path.exists(buffered_layer_path): + log_message( + f"Failed to create buffer for area {index}", + level=Qgis.Warning, + ) + return False + + buffered_layer = QgsVectorLayer(buffered_layer_path, "buffered", "ogr") + if not buffered_layer.isValid(): + log_message( + f"Failed to load buffered layer for area {index}", + level=Qgis.Warning, + ) + return False + + # Add value field with score 5 (highest accessibility) + field_names = [field.name() for field in buffered_layer.fields()] + if "value" not in field_names: + buffered_layer.dataProvider().addAttributes([QgsField("value", QVariant.Int)]) + buffered_layer.updateFields() + + buffered_layer.startEditing() + for feature in buffered_layer.getFeatures(): + feature.setAttribute("value", 5) + buffered_layer.updateFeature(feature) + buffered_layer.commitChanges() + + return buffered_layer + def _clip_network_to_area( self, clip_geometry: QgsGeometry, @@ -240,13 +423,9 @@ def _clip_network_to_area( """ buffer_distance = max(self.distances) if self.distances else 5000 buffered_geometry = clip_geometry.buffer(buffer_distance, 5) - bbox = buffered_geometry.boundingBox() - clipped_network_path = os.path.join(self.workflow_directory, f"clipped_network_area_{area_index}.gpkg") - if os.path.exists(clipped_network_path): os.remove(clipped_network_path) - try: road_network_layer = QgsVectorLayer(self.road_network_layer_path, "network", "ogr") if not road_network_layer.isValid(): @@ -255,13 +434,11 @@ def _clip_network_to_area( level=Qgis.Critical, ) return None - road_crs = road_network_layer.crs() log_message( f"Area {area_index}: Road network CRS: {road_crs.authid()}, Target CRS: {self.target_crs.authid()}", level=Qgis.Info, ) - # Auto-reproject road network if CRS mismatch detected if road_crs != self.target_crs: log_message( @@ -269,7 +446,6 @@ def _clip_network_to_area( f"{road_crs.authid()} to {self.target_crs.authid()}", level=Qgis.Info, ) - # Reproject to target CRS in memory (consistent with check_and_reproject_layer behavior) try: reproject_result = processing.run( @@ -282,14 +458,12 @@ def _clip_network_to_area( context=self.context, ) road_network_layer = reproject_result["OUTPUT"] - if not road_network_layer.isValid(): log_message( f"ERROR: Failed to reproject road network for area {area_index}", level=Qgis.Critical, ) return None - log_message( f"Successfully reprojected road network to {self.target_crs.authid()}", level=Qgis.Info, @@ -300,20 +474,16 @@ def _clip_network_to_area( level=Qgis.Critical, ) return None - log_message( f"Clipping road network to area {area_index} with {buffer_distance}m buffer", level=Qgis.Info, ) - temp_layer = QgsVectorLayer(f"Polygon?crs={self.target_crs.authid()}", "clip_geometry", "memory") temp_provider = temp_layer.dataProvider() - temp_feature = QgsFeature() temp_feature.setGeometry(buffered_geometry) temp_provider.addFeatures([temp_feature]) temp_layer.updateExtents() - # Use road_network_layer (potentially reprojected) instead of path result = processing.run( "native:clip", @@ -324,15 +494,12 @@ def _clip_network_to_area( }, context=self.context, ) - clipped_layer = result["OUTPUT"] - if isinstance(clipped_layer, str): check_layer = QgsVectorLayer(clipped_layer, "check", "ogr") feature_count = check_layer.featureCount() else: feature_count = clipped_layer.featureCount() - if feature_count == 0: log_message( f"Warning: Clipped network for area {area_index} has no features. " @@ -340,14 +507,11 @@ def _clip_network_to_area( level=Qgis.Warning, ) return None - log_message( f"Successfully clipped network to area {area_index}: {feature_count} road segments", level=Qgis.Info, ) - return clipped_network_path - except Exception as e: log_message( f"Error clipping network for area {area_index}: {e}", @@ -375,13 +539,11 @@ def create_isochrones( if total_features == 0: log_message(f"No features to process for area {area_index}.") return False - point_crs = point_layer.crs() log_message( f"Area {area_index}: Point layer CRS: {point_crs.authid()}, Target CRS: {self.target_crs.authid()}", level=Qgis.Info, ) - if point_crs != self.target_crs: log_message( f"ERROR: CRS mismatch for area {area_index}! " @@ -390,21 +552,17 @@ def create_isochrones( level=Qgis.Critical, ) return False - isochrone_layer_path = os.path.join(self.workflow_directory, f"isochrones_area_{area_index}.gpkg") - clipped_network_path = self._clip_network_to_area( clip_geometry=clip_geometry, area_index=area_index, ) - if not clipped_network_path: log_message( f"No road network available for area {area_index}. Skipping network analysis.", level=Qgis.Warning, ) return False - task = NativeNetworkAnalysisProcessingTask( point_layer=point_layer, distances=self.distances, @@ -412,10 +570,8 @@ def create_isochrones( output_gpkg_path=isochrone_layer_path, target_crs=self.target_crs, ) - task.progressChanged.connect(lambda progress: self.feedback.setProgress(progress)) success = task.run() - if os.path.exists(clipped_network_path): try: os.remove(clipped_network_path) @@ -428,7 +584,6 @@ def create_isochrones( f"Warning: Failed to clean up clipped network {clipped_network_path}: {e}", level=Qgis.Warning, ) - if not success: error_msg = task.error_message or "Unknown error" log_message( @@ -436,23 +591,25 @@ def create_isochrones( level=Qgis.Warning, ) return False - # Return the path to the created GeoPackage return task.result_path - def _process_features_with_simple_buffer( + def _process_features_with_simple_buffer_legacy( self, - current_area: "QgsGeometry", - clip_area: "QgsGeometry", - current_bbox: "QgsGeometry", - area_features: "QgsVectorLayer", + current_area: QgsGeometry, + clip_area: QgsGeometry, + current_bbox: QgsGeometry, + area_features: QgsVectorLayer, index: int, + area_name: str = None, ) -> str: """Process a single area using simple buffer (no network analysis). Used for Regional scale - creates a single buffer around POIs and scores grid cells based on percentage intersection. + This is the legacy raster-first version. + Args: current_area: Polygon from study area. clip_area: Polygon to clip features to. @@ -463,18 +620,15 @@ def _process_features_with_simple_buffer( Returns: Raster file path, or False if failed. """ - from qgis import processing from geest.core.algorithms.utilities import subset_vector_layer log_message( f"Creating simple buffer for area {index + 1} (buffer: {self.buffer_distance}m)", level=Qgis.Info, ) - buffer_output = os.path.join(self.workflow_directory, f"simple_buffer_{index}.gpkg") if os.path.exists(buffer_output): os.remove(buffer_output) - buffer_params = { "INPUT": area_features, "DISTANCE": self.buffer_distance, @@ -482,14 +636,12 @@ def _process_features_with_simple_buffer( } result = processing.run("native:buffer", buffer_params, feedback=QgsProcessingFeedback()) buffered_layer_path = result["OUTPUT"] - if not buffered_layer_path or not os.path.exists(buffered_layer_path): log_message( f"Failed to create buffer for area {index}", level=Qgis.Warning, ) return False - buffered_layer = QgsVectorLayer(buffered_layer_path, "buffered", "ogr") if not buffered_layer.isValid(): log_message( @@ -497,11 +649,9 @@ def _process_features_with_simple_buffer( level=Qgis.Warning, ) return False - grid_output = os.path.join(self.workflow_directory, f"grid_area_{index}.gpkg") if os.path.exists(grid_output): os.remove(grid_output) - area_grid = subset_vector_layer( self.workflow_directory, self.grid_layer, @@ -514,33 +664,29 @@ def _process_features_with_simple_buffer( level=Qgis.Warning, ) return False - scored_grid = self._score_grid_for_percentage( grid_layer=area_grid, buffered_layer=buffered_layer, ) - if scored_grid is False: log_message( "No scored grid cells were created.", level=Qgis.Warning, ) return False - raster_output = self._rasterize( input_layer=scored_grid, bbox=current_bbox, index=index, value_field="value", ) - return raster_output def _score_grid_for_percentage( self, - grid_layer: "QgsVectorLayer", - buffered_layer: "QgsVectorLayer", - ) -> "QgsVectorLayer": + grid_layer: QgsVectorLayer, + buffered_layer: QgsVectorLayer, + ) -> QgsVectorLayer: """Score grid cells based on percentage intersection with buffered features. For Regional scale: calculates what percentage of each grid cell @@ -554,44 +700,35 @@ def _score_grid_for_percentage( The grid layer with "value" field containing assigned scores. """ log_message("Scoring grid cells based on percentage intersection") - field_names = [field.name() for field in grid_layer.fields()] if "value" not in field_names: grid_layer.dataProvider().addAttributes([QgsField("value", QVariant.Int)]) grid_layer.updateFields() - grid_layer.startEditing() for grid_feature in grid_layer.getFeatures(): grid_geom = grid_feature.geometry() if grid_geom.isNull(): continue - grid_area = grid_geom.area() if grid_area == 0: continue - max_score = 0 max_overlap_percent = 0 - for buffered_feature in buffered_layer.getFeatures(): buffered_geom = buffered_feature.geometry() if buffered_geom.isNull(): continue - intersection = grid_geom.intersection(buffered_geom) if intersection.isNull() or intersection.area() == 0: continue - # Calculate % of buffer within hexagon (not % of hexagon covered) buffer_area = buffered_geom.area() if buffer_area > 0: overlap_percent = (intersection.area() / buffer_area) * 100 else: overlap_percent = 0 - if overlap_percent > max_overlap_percent: max_overlap_percent = overlap_percent - # Calculate score based on final max_overlap_percent after all buffers checked # Table ranges: Score 0: 0%, Score 1: 0.01-6%, Score 2: 6.01-12%, etc. sorted_items = sorted(self.percentage_scores.items()) @@ -628,15 +765,12 @@ def _score_grid_for_percentage( ) if in_range: max_score = score - log_message( f"DEBUG: Feature {grid_feature.id()} - FINAL: max_overlap={max_overlap_percent:.4f}%, assigned_score={max_score}", level=Qgis.Info, ) - grid_feature.setAttribute("value", max_score) grid_layer.updateFeature(grid_feature) - grid_layer.commitChanges() return grid_layer @@ -658,21 +792,17 @@ def _create_bands(self, isochrones_gpkg_path, index): KeyError: If the value field does not exist in the isochrone layer. """ isochrone_layer_path = f"{isochrones_gpkg_path}|layername=isochrones" - layer = QgsVectorLayer(isochrone_layer_path, "isochrones", "ogr") if not layer.isValid(): raise ValueError(f"Failed to load isochrone layer from {isochrone_layer_path}") output_path = os.path.join(self.workflow_directory, f"final_isochrones_{index}.shp") - ranges_field = "value" field_index = layer.fields().indexFromName(ranges_field) if field_index == -1: raise KeyError( f"Field '{ranges_field}' does not exist in isochrones layer: {isochrone_layer_path}" # noqa E713 ) - unique_ranges = sorted(self.distances, reverse=False) - range_layers = {} for value in unique_ranges: expression = f'"value" = {value}' @@ -685,7 +815,6 @@ def _create_bands(self, isochrones_gpkg_path, index): data_provider.addAttributes(layer.fields()) range_layer.updateFields() data_provider.addFeatures(features) - dissolve_params = { "INPUT": range_layer, "FIELD": [], @@ -694,7 +823,6 @@ def _create_bands(self, isochrones_gpkg_path, index): dissolve_result = processing.run("native:dissolve", dissolve_params) dissolved_layer = dissolve_result["OUTPUT"] range_layers[value] = dissolved_layer - band_layers = [] sorted_ranges = sorted(range_layers.keys(), reverse=True) for i in range(len(sorted_ranges) - 1): @@ -702,7 +830,6 @@ def _create_bands(self, isochrones_gpkg_path, index): next_range = sorted_ranges[i + 1] current_layer = range_layers[current_range] next_layer = range_layers[next_range] - difference_params = { "INPUT": current_layer, "OVERLAY": next_layer, @@ -710,7 +837,6 @@ def _create_bands(self, isochrones_gpkg_path, index): } diff_result = processing.run("native:difference", difference_params) diff_layer = diff_result["OUTPUT"] - diff_layer.dataProvider().addAttributes( [ QgsField("distance", QVariant.Int), @@ -721,14 +847,11 @@ def _create_bands(self, isochrones_gpkg_path, index): for feat in diff_layer.getFeatures(): feat["distance"] = current_range diff_layer.updateFeature(feat) - band_layers.append(diff_layer) - try: smallest_range = sorted_ranges[-1] except IndexError: return None - smallest_layer = range_layers[smallest_range] smallest_layer.dataProvider().addAttributes([QgsField("distance", QVariant.Int)]) smallest_layer.updateFields() @@ -737,7 +860,6 @@ def _create_bands(self, isochrones_gpkg_path, index): feat["distance"] = smallest_range smallest_layer.updateFeature(feat) band_layers.append(smallest_layer) - merge_bands_params = { "LAYERS": band_layers, "CRS": self.target_crs, @@ -759,7 +881,6 @@ def _assign_scores(self, layer: QgsVectorLayer) -> QgsVectorLayer: """ if not layer or not layer.isValid(): return False - # Check if the "value" field already exists field_names = [field.name() for field in layer.fields()] log_message(f"Field names: {field_names}") @@ -768,17 +889,15 @@ def _assign_scores(self, layer: QgsVectorLayer) -> QgsVectorLayer: layer.dataProvider().addAttributes([QgsField("value", QVariant.Int)]) layer.updateFields() log_message('Added "value" field to input layer') - # Check if we should use percentage-based scoring (Regional scale) if self.scoring_method == "percentage_intersection" and self.percentage_scores: layer = self._assign_percentage_scores(layer) else: # Original distance-based scoring layer = self._assign_distance_scores(layer) - return layer - def _assign_percentage_scores(self, layer: "QgsVectorLayer") -> "QgsVectorLayer": + def _assign_percentage_scores(self, layer: QgsVectorLayer) -> QgsVectorLayer: """Assign scores based on percentage intersection with buffer. For Regional scale: calculates what percentage of each grid cell @@ -791,7 +910,6 @@ def _assign_percentage_scores(self, layer: "QgsVectorLayer") -> "QgsVectorLayer" The same layer with a "value" field containing the assigned scores. """ log_message("Using percentage-based scoring for Regional scale") - buffer_distance = 0 if hasattr(self, "buffer_distances") and self.buffer_distances: buffer_distance = ( @@ -799,46 +917,37 @@ def _assign_percentage_scores(self, layer: "QgsVectorLayer") -> "QgsVectorLayer" ) elif self.distances: buffer_distance = max(self.distances) if isinstance(self.distances, list) else self.distances - log_message(f"Buffer distance for percentage scoring: {buffer_distance}m") - layer.startEditing() for feature in layer.getFeatures(): grid_geom = feature.geometry() if grid_geom.isNull(): continue - grid_area = grid_geom.area() - buffer_geom = feature.geometry() if buffer_geom.isNull(): continue - intersection = grid_geom.intersection(buffer_geom) if intersection.isNull() or intersection.area() == 0: feature.setAttribute("value", 0) else: overlap_percent = (intersection.area() / grid_area) * 100 - score = 0 for min_pct, score_value in sorted(self.percentage_scores.items(), reverse=True): if overlap_percent >= min_pct: score = score_value break - feature.setAttribute("value", score) log_message( f"Grid cell overlap: {overlap_percent:.2f}%, score: {score}", tag="GeoE3", level=Qgis.Info, ) - layer.updateFeature(feature) - layer.commitChanges() return layer - def _assign_distance_scores(self, layer: "QgsVectorLayer") -> "QgsVectorLayer": + def _assign_distance_scores(self, layer: QgsVectorLayer) -> QgsVectorLayer: """Assign scores based on distance band (original method). Args: @@ -876,6 +985,7 @@ def _process_raster_for_area( current_bbox: QgsGeometry, area_raster: str, index: int, + area_name: str = None, ): """Execute the actual workflow logic for a single area using a raster. @@ -896,6 +1006,7 @@ def _process_aggregate_for_area( clip_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: str = None, ): """Execute the workflow, reporting progress and checking for cancellation. diff --git a/geest/core/workflows/multi_buffer_distances_ors_workflow.py b/geest/core/workflows/multi_buffer_distances_ors_workflow.py index 66705fae..7290ad44 100644 --- a/geest/core/workflows/multi_buffer_distances_ors_workflow.py +++ b/geest/core/workflows/multi_buffer_distances_ors_workflow.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- """📦 Multi Buffer Distances Ors Workflow module. - This module contains functionality for multi buffer distances ors workflow. """ - import os import traceback from urllib.parse import unquote @@ -36,19 +34,13 @@ class MultiBufferDistancesORSWorkflow(WorkflowBase): """ Concrete implementation of a 'multi_buffer_distances' workflow. - This uses ORS (OpenRouteService) to calculate the distances between the study area and the selected points of interest. - It will create concentric buffers (isochrones) around the study area and calculate the distances to the points of interest. - The buffers will be calcuated either using travel time or travel distance. - The results will be stored as a collection of tif files scaled to the likert scale. - These results will be be combined into a VRT file and added to the QGIS map. - """ def __init__( @@ -113,7 +105,6 @@ def __init__( level=Qgis.Warning, ) raise Exception("Invalid travel distances provided.") - layer_path = self.attributes.get("multi_buffer_point_shapefile", None) if layer_path: layer_path = unquote(layer_path) @@ -140,7 +131,6 @@ def __init__( level=Qgis.Warning, ) raise Exception("Invalid points layer found.") - mode = self.attributes.get("multi_buffer_travel_mode", "Walking") self.mode = None if mode == "Walking": @@ -153,13 +143,11 @@ def __init__( self.measurement = "distance" else: self.measurement = "time" - # How many features to pass with each ORS API call # Managed in the settings panel self.subset_size = int(setting(key="ors_request_size", default=5)) if self.subset_size > 5: self.subset_size = 5 # Maxiumum of 5 features per request allowed by ORS - self.ors_client = ORSClient("https://api.openrouteservice.org/v2/isochrones") self.api_key = self.ors_client.check_api_key() # Create the masked API key for logging @@ -172,16 +160,13 @@ def __init__( def _mask_api_key(self, api_key: str) -> str: """ Safely mask an API key for logging purposes. - Args: api_key (str): The API key to mask - Returns: str: The masked API key """ if not api_key: return "****" - key_len = len(api_key) if key_len <= 8: # For short keys, show only asterisks @@ -197,38 +182,32 @@ def _process_features_for_area( current_bbox: QgsGeometry, area_features: QgsVectorLayer, index: int, + area_name: str = None, ) -> str: """ Executes the actual workflow logic for a single area. Must be implemented by subclasses. - :current_area: Current polygon from our study area. :current_bbox: Bounding box of the above area. :area_features: A vector layer of features to analyse that includes only features in the study area. :index: Iteration / number of area being processed. - :return: A raster layer file path if processing completes successfully, False if canceled or failed. """ - # Step 2: Process these areas in batches and create buffers buffers = self.create_multibuffers( point_layer=area_features, index=index, ) - scored_buffers = self._assign_scores(buffers) - if scored_buffers is False: log_message("No scored buffers were created.", level=Qgis.Warning) return False - raster_output = self._rasterize( input_layer=scored_buffers, bbox=current_bbox, index=index, value_field="value", ) - return raster_output def create_multibuffers( @@ -238,28 +217,23 @@ def create_multibuffers( ): """ Create multiple buffers (isochrones) for each point in the input point layer using ORSClient. - This method processes the point features in subsets (to handle large datasets), makes API calls to the OpenRouteService to fetch the isochrones (buffers) for each subset, and merges the results into a final output layer. - :param point_layer: QgsVectorLayer containing point features to process. :param index: Index of the current area being processed. :return: QgsVectorLayer containing the buffers as polygons. """ # codeql[python/clear-text-logging-sensitive-data] - API key is properly masked before logging log_message(f"Using ORS API key: {self.masked_api_key}") - # Collect intermediate layers from ORS API features = list(point_layer.getFeatures()) log_message(f"Creating buffers for {len(features)} points") total_features = len(features) - # Process features in subsets to handle large datasets for i in range(0, total_features, self.subset_size): subset_features = features[i : i + self.subset_size] # noqa E203 subset_layer = self._create_subset_layer(subset_features, point_layer) - # Make API calls using ORSClient for the subset json = self._fetch_isochrones(subset_layer) layer = self._create_isochrone_layer(json) @@ -270,7 +244,6 @@ def create_multibuffers( tag="GeoE3", level=Qgis.Info, ) - # Merge all isochrone layers into one final output if self.temp_layers: log_message( @@ -299,53 +272,41 @@ def _create_subset_layer(self, subset_features, point_layer): """ Create a subset layer for processing, with reprojection of points from the point_layer CRS to EPSG:4326 (WGS 84). - :param subset_features: List of QgsFeature objects to add to the subset layer. :param point_layer: The original point layer (QgsVectorLayer) to reproject from. :return: A QgsVectorLayer (subset layer) with reprojected features. """ target_crs = QgsCoordinateReferenceSystem("EPSG:4326") - # Create a new memory layer with the target CRS (EPSG:4326) subset_layer = QgsVectorLayer(f"Point?crs={target_crs.authid()}", "subset", "memory") subset_layer_data = subset_layer.dataProvider() - # Add attributes (fields) from the point_layer subset_layer_data.addAttributes(point_layer.fields()) subset_layer.updateFields() - # Create coordinate transformation from point_layer CRS to the target CRS (EPSG:4326) source_crs = point_layer.crs() transform_context = self.context.project().transformContext() transform = QgsCoordinateTransform(source_crs, target_crs, transform_context) - # Reproject and add features to the subset layer reprojected_features = [] for feature in subset_features: reprojected_feature = QgsFeature(feature) geom = reprojected_feature.geometry() - # Transform the geometry to the target CRS geom.transform(transform) reprojected_feature.setGeometry(geom) - reprojected_features.append(reprojected_feature) - # Add reprojected features to the new subset layer subset_layer_data.addFeatures(reprojected_features) - return subset_layer def _fetch_isochrones(self, layer: QgsVectorLayer) -> dict: """ Fetch isochrones for the given subset of features using ORSClient. - Args: layer (QgsVectorLayer): A QgsVectorLayer containing the subset of features. - Returns: dict: A dict representing the JSON response from the ORS API. - Raises: ValueError: If no valid coordinates are found in the layer. Any exceptions raised by ORSClient.make_request will propagate. @@ -357,17 +318,14 @@ def _fetch_isochrones(self, layer: QgsVectorLayer) -> dict: if geom and not geom.isMultipart(): # Single point geometry coords = geom.asPoint() coordinates.append([coords.x(), coords.y()]) - if not coordinates: raise ValueError("No valid coordinates found in the layer") - # Prepare parameters for ORS API params = { "locations": coordinates, "range": self.distances, # Distances or times in the list "range_type": self.measurement, } - # Make the request to ORS API using ORSClient # Any exceptions will be propogated try: @@ -381,7 +339,6 @@ def _fetch_isochrones(self, layer: QgsVectorLayer) -> dict: with open(error_path, "w") as f: f.write(f"Failed to process {self.workflow_name}: {e}\n") f.write(traceback.format_exc()) - log_message( f"Failed to fetch isochrones layer for {self.workflow_name}: {e}", tag="GeoE3", @@ -402,18 +359,15 @@ def _fetch_isochrones(self, layer: QgsVectorLayer) -> dict: def _create_isochrone_layer(self, isochrone_data): """ Create a QgsVectorLayer from the ORS isochrone data. - :param isochrone_data: JSON data returned from ORS. :return: A QgsVectorLayer containing the isochrones as polygons. """ isochrone_layer = QgsVectorLayer("Polygon?crs=EPSG:4326", "isochrones", "memory") provider = isochrone_layer.dataProvider() - # Add the 'value' field to the layer's attribute table isochrone_layer.startEditing() isochrone_layer.addAttribute(QgsField("value", QVariant.Int)) isochrone_layer.commitChanges() - # Parse the features from ORS response verbose_mode = int(setting(key="verbose_mode", default=0)) if isochrone_data and "features" in isochrone_data: @@ -447,18 +401,15 @@ def _create_isochrone_layer(self, isochrone_data): feat.setGeometry(qgs_geometry) feat.setAttributes([feature_data["properties"].get("value", 0)]) # Add attributes as needed features.append(feat) - provider.addFeatures(features) return isochrone_layer def _merge_layers(self, layers=None, index=None): """ Merge all temporary isochrone layers into a single layer. - :param layers: List of temporary QgsVectorLayers to merge. :param crs: The CRS to use for the merged layer. :param index: The index of the current area being processed. - :return: A QgsVectorLayer representing the merged isochrone layers. """ merge_output = os.path.join(self.workflow_directory, f"{self.layer_id}_merged_isochrones_{index}.shp") @@ -474,26 +425,20 @@ def _merge_layers(self, layers=None, index=None): def _create_bands(self, layer, index): """ Create bands by computing differences between isochrone ranges. - This method computes the differences between isochrone ranges to create bands of non overlapping polygons. The bands are then merged into a final output layer. - :param layer: The merged isochrone layer. :param crs: Coordinate reference system for the output. :param index: The index of the current area being processed. - Returns: QgsVectoryLayer: The final output QgsVectorLayer layer path containing the bands. """ output_path = os.path.join(self.workflow_directory, f"final_isochrones_{index}.shp") - ranges_field = "value" field_index = layer.fields().indexFromName(ranges_field) if field_index == -1: raise KeyError(f"Field '{ranges_field}' does not exist in the merged layer.") # noqa E713 - unique_ranges = sorted({feat[ranges_field] for feat in layer.getFeatures()}) - range_layers = {} for r in unique_ranges: expr = f'"{ranges_field}" = {r}' @@ -506,7 +451,6 @@ def _create_bands(self, layer, index): dp.addAttributes(layer.fields()) range_layer.updateFields() dp.addFeatures(features) - dissolve_params = { "INPUT": range_layer, "FIELD": [], @@ -515,7 +459,6 @@ def _create_bands(self, layer, index): dissolve_result = processing.run("native:dissolve", dissolve_params) dissolved_layer = dissolve_result["OUTPUT"] range_layers[r] = dissolved_layer - band_layers = [] sorted_ranges = sorted(range_layers.keys(), reverse=True) for i in range(len(sorted_ranges) - 1): @@ -523,7 +466,6 @@ def _create_bands(self, layer, index): next_range = sorted_ranges[i + 1] current_layer = range_layers[current_range] next_layer = range_layers[next_range] - difference_params = { "INPUT": current_layer, "OVERLAY": next_layer, @@ -531,7 +473,6 @@ def _create_bands(self, layer, index): } diff_result = processing.run("native:difference", difference_params) diff_layer = diff_result["OUTPUT"] - diff_layer.dataProvider().addAttributes( [ QgsField("distance", QVariant.Int), @@ -542,9 +483,7 @@ def _create_bands(self, layer, index): for feat in diff_layer.getFeatures(): feat["distance"] = current_range diff_layer.updateFeature(feat) - band_layers.append(diff_layer) - smallest_range = sorted_ranges[-1] smallest_layer = range_layers[smallest_range] smallest_layer.dataProvider().addAttributes([QgsField("distance", QVariant.Int)]) @@ -554,7 +493,6 @@ def _create_bands(self, layer, index): feat["distance"] = smallest_range smallest_layer.updateFeature(feat) band_layers.append(smallest_layer) - merge_bands_params = { "LAYERS": band_layers, "CRS": self.target_crs, @@ -569,16 +507,13 @@ def _create_bands(self, layer, index): def _assign_scores(self, layer: QgsVectorLayer) -> QgsVectorLayer: """ Assign values to buffered polygons based 5 for presence of a polygon. - Args: layer QgsVectorLayer: The buffered features layer. - Returns: QgsVectorLayer: The same layer with a "value" field containing the assigned scores. """ if not layer or not layer.isValid(): return False - # Check if the "value" field already exists field_names = [field.name() for field in layer.fields()] log_message(f"Field names: {field_names}") @@ -587,10 +522,8 @@ def _assign_scores(self, layer: QgsVectorLayer) -> QgsVectorLayer: # Add the burn field to the input layer if it doesn't exist layer.dataProvider().addAttributes([QgsField("value", QVariant.Int)]) layer.updateFields() - # Log message when the field is added log_message('Added "value" field to input layer') - # Calculate the burn field value based on the item number in the distance list layer.startEditing() for i, feature in enumerate(layer.getFeatures()): @@ -616,16 +549,12 @@ def _assign_scores(self, layer: QgsVectorLayer) -> QgsVectorLayer: def reproject_isochrones(self, layer: QgsVectorLayer): """ Reproject the isochrone layer to target crs. - The resulting layer will be saved in the working directory too. - Parameters: layer (QgsVectorLayer): The input isochrone layer to reproject. - Returns: QgsVectorLayer: The reprojected isochrone layer. """ - # reproject the later to self.target_crs input_path = layer.source() reprojected_layer_path = input_path.replace(".shp", f"_epsg{self.target_crs.postgisSrid()}.shp") @@ -641,7 +570,6 @@ def reproject_isochrones(self, layer: QgsVectorLayer): ) reprojected_layer_result = processing.run("native:reprojectlayer", transform_params) reprojected_layer = QgsVectorLayer(reprojected_layer_result["OUTPUT"], "reprojected_layer", "ogr") - if not reprojected_layer.isValid(): raise ValueError(f"Failed to reproject input layer to {self.target_crs.authid()}") return reprojected_layer @@ -654,16 +582,15 @@ def _process_raster_for_area( current_bbox: QgsGeometry, area_raster: str, index: int, + area_name: str = None, ): """ Executes the actual workflow logic for a single area using a raster. - :current_area: Current polygon from our study area. :clip_area: Polygon to clip the raster to which is aligned to cell edges. :current_bbox: Bounding box of the above area. :area_raster: A raster layer of features to analyse that includes only bbox pixels in the study area. :index: Index of the current area. - :return: Path to the reclassified raster. """ pass @@ -674,6 +601,7 @@ def _process_aggregate_for_area( clip_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: str = None, ): """ Executes the workflow, reporting progress through the feedback object and checking for cancellation. diff --git a/geest/core/workflows/osm_transport_polyline_per_cell_workflow.py b/geest/core/workflows/osm_transport_polyline_per_cell_workflow.py index cdf46b76..62ac9c01 100644 --- a/geest/core/workflows/osm_transport_polyline_per_cell_workflow.py +++ b/geest/core/workflows/osm_transport_polyline_per_cell_workflow.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- """📦 Osm Transport Polyline Per Cell Workflow module. - This module contains functionality for osm transport polyline per cell workflow. """ - import os from typing import Optional from urllib.parse import unquote @@ -40,7 +38,6 @@ def __init__( ): """ Initialize the workflow with attributes and feedback. - Args: :param item: JsonTreeItem representing the analysis, dimension, or factor to process. :param cell_size_m: Cell size in meters @@ -53,14 +50,11 @@ def __init__( item, cell_size_m, analysis_scale, feedback, context, working_directory ) # ⭐️ Item is a reference - whatever you change in this item will directly update the tree self.workflow_name = "use_osm_transport_polyline_per_cell" - # Use unified active transport - combines both highway and cycleway with best score logic self.osm_processing_type = OSMDownloadType.ACTIVE_TRANSPORT - layer_path = self.attributes.get("osm_transport_polyline_per_cell_shapefile", None) if layer_path: layer_path = unquote(layer_path) - if not layer_path: log_message( "Nothing found in osm_transport_polyline_per_cell_shapefile, trying osm_transport_polyline_per_cell_layer_source.", @@ -68,7 +62,6 @@ def __init__( level=Qgis.Warning, ) layer_path = self.attributes.get("osm_transport_polyline_per_cell_layer_source", None) - if not layer_path: log_message( "No osm_transport_polyline_per_cell_layer_source found, trying road_network_layer_path.", @@ -76,12 +69,10 @@ def __init__( level=Qgis.Warning, ) layer_path = self.attributes.get("road_network_layer_path", None) - if not layer_path: error_msg = "No transport layer found. Please configure a data source or download the active transport network." log_message(error_msg, tag="GeoE3", level=Qgis.Critical) raise ValueError(error_msg) - self.features_layer = QgsVectorLayer(layer_path, "OSM Transport Layer", "ogr") def _process_features_for_area( @@ -91,16 +82,15 @@ def _process_features_for_area( current_bbox: QgsGeometry, area_features: QgsVectorLayer, index: int, + area_name: str = None, ) -> str: """ Executes the actual workflow logic for a single area Must be implemented by subclasses. - :current_area: Current polygon from our study area. :current_bbox: Bounding box of the above area. :area_features: A vector layer of features to analyse that includes only features in the study area. :index: Iteration / number of area being processed. - :return: A raster layer file path if processing completes successfully, False if canceled or failed. """ area_features_count = area_features.featureCount() @@ -120,14 +110,12 @@ def _process_features_for_area( self.feedback, analysis_scale=self.analysis_scale, ) - log_message( "OSM Transport Polyline per Cell - Selected grid cells and assigned transport scores.", tag="GeoE3", level=Qgis.Info, ) log_message(f"Grid cells with transport scores saved to: {output_path}", tag="GeoE3", level=Qgis.Info) - # Step 2: Rasterize the grid layer using the assigned values # Create a scored boundary layer self.updateStatus("Rasterizing grid cells...") @@ -148,16 +136,15 @@ def _process_raster_for_area( current_bbox: QgsGeometry, area_raster: str, index: int, + area_name: str = None, ): """ Executes the actual workflow logic for a single area using a raster. - :current_area: Current polygon from our study area. :clip_area: Polygon to clip the raster to which is aligned to cell edges. :current_bbox: Bounding box of the above area. :area_raster: A raster layer of features to analyse that includes only bbox pixels in the study area. :index: Index of the current area. - :return: Path to the reclassified raster. """ pass @@ -168,6 +155,7 @@ def _process_aggregate_for_area( clip_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: str = None, ): """ Executes the workflow, reporting progress through the feedback object and checking for cancellation. diff --git a/geest/core/workflows/point_per_cell_workflow.py b/geest/core/workflows/point_per_cell_workflow.py index c47702c4..55e6c0ea 100644 --- a/geest/core/workflows/point_per_cell_workflow.py +++ b/geest/core/workflows/point_per_cell_workflow.py @@ -2,9 +2,13 @@ """📦 Point Per Cell Workflow module. This module contains functionality for point per cell workflow. + +Supports grid-first mode where feature counts are written directly to +the study_area_grid column, then rasterized. """ import os +from typing import Optional from urllib.parse import unquote from qgis.core import ( @@ -16,9 +20,10 @@ ) from geest.core import JsonTreeItem -from geest.core.algorithms.features_per_cell_processor import ( - assign_values_to_grid, - select_grid_cells_and_count_features, +from geest.core.grid_column_utils import ( + clear_grid_column, + count_features_per_grid_cell, + rasterize_grid_column, ) from geest.utilities import log_message @@ -86,7 +91,12 @@ def __init__( self.attributes["result"] = f"{self.workflow_name} Workflow Failed" raise Exception(error) - self.feedback.setProgress(1.0) # We just use nominal intervals for progress updates + self.feedback.setProgress(1.0) + self.workflow_name = "point_per_cell" + # Grid-first mode: write results to grid columns first, then rasterize + self.use_grid_first = True + # Track if we've cleared the column (only do once, not per area) + self._column_cleared = False def _process_features_for_area( self, @@ -95,17 +105,20 @@ def _process_features_for_area( current_bbox: QgsGeometry, area_features: QgsVectorLayer, index: int, + area_name: Optional[str] = None, ) -> str: """ - Executes the actual workflow logic for a single area - Must be implemented by subclasses. + Executes the actual workflow logic for a single area. + + Supports grid-first mode where counts are written directly to study_area_grid. :current_area: Current polygon from our study area. :current_bbox: Bounding box of the above area. - :area_features: A vector layer of features to analyse that includes only features in the study area. + :area_features: A vector layer of features to analyse. :index: Iteration / number of area being processed. + :area_name: Name of the area being processed. - :return: A raster layer file path if processing completes successfully, False if canceled or failed. + :return: A raster layer file path if processing completes successfully. """ area_features_count = area_features.featureCount() log_message( @@ -113,15 +126,36 @@ def _process_features_for_area( tag="GeoE3", level=Qgis.Info, ) - # Step 1: Select grid cells that intersect with features + + if self.use_grid_first: + return self._process_grid_first( + current_bbox=current_bbox, + area_features=area_features, + index=index, + area_name=area_name, + ) + else: + return self._process_raster_first( + current_bbox=current_bbox, + area_features=area_features, + index=index, + ) + + def _process_raster_first( + self, + current_bbox: QgsGeometry, + area_features: QgsVectorLayer, + index: int, + ) -> str: + """Legacy raster-first processing using copied grid.""" + from geest.core.algorithms.features_per_cell_processor import ( + assign_values_to_grid, + select_grid_cells_and_count_features, + ) + output_path = os.path.join(self.workflow_directory, f"{self.layer_id}_grid_cells.gpkg") area_grid = select_grid_cells_and_count_features(self.grid_layer, area_features, output_path, self.feedback) - - # Step 2: Assign values to grid cells grid = assign_values_to_grid(area_grid, self.feedback) - - # Step 3: Rasterize the grid layer using the assigned values - # Create a scored boundary layer raster_output = self._rasterize( grid, current_bbox, @@ -131,6 +165,56 @@ def _process_features_for_area( ) return raster_output + def _process_grid_first( + self, + current_bbox: QgsGeometry, + area_features: QgsVectorLayer, + index: int, + area_name: str, + ) -> str: + """Grid-first processing - writes directly to study_area_grid.""" + # Clear column once at the start (not per area) + if not self._column_cleared: + log_message(f"Clearing column {self.layer_id} before processing") + clear_grid_column(self.gpkg_path, self.layer_id) + self._column_cleared = True + + self.progressChanged.emit(10.0) + + # Count features and write to grid + log_message(f"Counting features for column {self.layer_id}") + count_features_per_grid_cell( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + features_layer=area_features, + feedback=self.feedback, + ) + + self.progressChanged.emit(50.0) + + # Rasterize from grid column + output_path = os.path.join( + self.workflow_directory, + f"{self.layer_id}_{index}.tif", + ) + + rect = current_bbox.boundingBox() + extent = (rect.xMinimum(), rect.yMinimum(), rect.xMaximum(), rect.yMaximum()) + + rasterize_grid_column( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + output_raster_path=output_path, + cell_size=self.cell_size_m, + extent=extent, + nodata=-9999.0, + area_name=area_name, + ) + + self.progressChanged.emit(100.0) + log_message(f"Rasterized grid column to {output_path}") + return output_path + # Default implementation of the abstract method - not used in this workflow def _process_raster_for_area( self, @@ -139,6 +223,7 @@ def _process_raster_for_area( current_bbox: QgsGeometry, area_raster: str, index: int, + area_name: str = None, ): """ Executes the actual workflow logic for a single area using a raster. @@ -159,6 +244,7 @@ def _process_aggregate_for_area( clip_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: str = None, ): """ Executes the workflow, reporting progress through the feedback object and checking for cancellation. diff --git a/geest/core/workflows/polygon_per_cell_workflow.py b/geest/core/workflows/polygon_per_cell_workflow.py index f7711748..262c15f4 100644 --- a/geest/core/workflows/polygon_per_cell_workflow.py +++ b/geest/core/workflows/polygon_per_cell_workflow.py @@ -2,9 +2,13 @@ """📦 Polygon Per Cell Workflow module. This module contains functionality for polygon per cell workflow. + +Supports grid-first mode where feature counts are written directly to +the study_area_grid column, then rasterized. """ import os +from typing import Optional from urllib.parse import unquote from qgis.core import ( @@ -16,8 +20,12 @@ ) from geest.core import JsonTreeItem -from geest.core.algorithms.polygon_per_cell_processor import ( - assign_reclassification_to_polygons, +from geest.core.constants import DEFAULT_S2S_EDUCATION_URBANIZATION_FIELDS +from geest.core.grid_column_utils import ( + clear_grid_column, + count_features_per_grid_cell, + rasterize_grid_column, + write_joined_values_to_grid, ) from geest.utilities import log_message @@ -50,13 +58,26 @@ def __init__( super().__init__( item, cell_size_m, analysis_scale, feedback, context, working_directory ) # ⭐️ Item is a reference - whatever you change in this item will directly update the tree - # TODO fix inconsistent abbreviation below for Poly self.workflow_name = "use_polygon_per_cell" + self.s2s_output_path = self.attributes.get("s2s_output_path", "") + self.s2s_fields = self._resolve_s2s_fields() + self._use_s2s_education_proxy = bool( + self.analysis_scale == "regional" + and self.layer_id == "education" + and self.s2s_output_path + and self.s2s_fields + ) + + if self._use_s2s_education_proxy: + self.features_layer = True + self.workflow_name = "polygon_per_cell" + self.use_grid_first = True + self._column_cleared = False + return layer_path = self.attributes.get("polygon_per_cell_shapefile", None) if layer_path: layer_path = unquote(layer_path) - if not layer_path: log_message( "Invalid raster found in polygon_per_cell_shapefile, trying polygon_per_cell_layer_source.", @@ -71,8 +92,12 @@ def __init__( level=Qgis.Warning, ) return False - self.features_layer = QgsVectorLayer(layer_path, "polygon_per_cell_layer", "ogr") + self.workflow_name = "polygon_per_cell" + # Grid-first mode: write results to grid columns first, then rasterize + self.use_grid_first = True + # Track if we've cleared the column (only do once, not per area) + self._column_cleared = False def _process_features_for_area( self, @@ -81,28 +106,59 @@ def _process_features_for_area( current_bbox: QgsGeometry, area_features: QgsVectorLayer, index: int, + area_name: Optional[str] = None, ) -> str: """ - Executes the actual workflow logic for a single area - Must be implemented by subclasses. + Executes the actual workflow logic for a single area. + + Supports grid-first mode where counts are written directly to study_area_grid. :current_area: Current polygon from our study area. :current_bbox: Bounding box of the above area. :area_features: A vector layer of features to analyse that includes only features in the study area. :index: Iteration / number of area being processed. + :area_name: Name of the area being processed. - :return: A raster layer file path if processing completes successfully, False if canceled or failed. + :return: A raster layer file path if processing completes successfully. """ - area_features_count = area_features.featureCount() - log_message( - f"Features layer for area {index + 1} loaded with {area_features_count} features.", - tag="GeoE3", - level=Qgis.Info, + if self.use_grid_first: + if self._use_s2s_education_proxy: + return self._process_s2s_education_proxy( + current_bbox=current_bbox, + index=index, + area_name=area_name, + ) + + area_features_count = area_features.featureCount() if area_features is not None else 0 + log_message( + f"Features layer for area {index + 1} loaded with {area_features_count} features.", + tag="GeoE3", + level=Qgis.Info, + ) + return self._process_grid_first( + current_bbox=current_bbox, + area_features=area_features, + index=index, + area_name=area_name, + ) + else: + return self._process_raster_first( + current_bbox=current_bbox, + area_features=area_features, + index=index, + ) + + def _process_raster_first( + self, + current_bbox: QgsGeometry, + area_features: QgsVectorLayer, + index: int, + ) -> str: + """Legacy raster-first processing using polygon perimeter classification.""" + from geest.core.algorithms.polygon_per_cell_processor import ( + assign_reclassification_to_polygons, ) - # Step 1: Select grid cells that intersect with features - output_path = os.path.join(self.workflow_directory, f"{self.layer_id}_grid_cells.gpkg") - del output_path - # Step 2: Assign reclassification values to polygons based on their perimeter + polygon_areas = assign_reclassification_to_polygons(area_features) raster_output = self._rasterize( polygon_areas, @@ -113,6 +169,197 @@ def _process_features_for_area( ) return raster_output + def _process_grid_first( + self, + current_bbox: QgsGeometry, + area_features: QgsVectorLayer, + index: int, + area_name: str, + ) -> str: + """Grid-first processing - writes directly to study_area_grid.""" + # Clear column once at the start (not per area) + if not self._column_cleared: + log_message(f"Clearing column {self.layer_id} before processing") + clear_grid_column(self.gpkg_path, self.layer_id) + self._column_cleared = True + + self.progressChanged.emit(10.0) + + # Count features and write to grid + log_message(f"Counting features for column {self.layer_id}") + count_features_per_grid_cell( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + features_layer=area_features, + feedback=self.feedback, + ) + + self.progressChanged.emit(50.0) + + # Rasterize from grid column + output_path = os.path.join( + self.workflow_directory, + f"{self.layer_id}_{index}.tif", + ) + + rect = current_bbox.boundingBox() + extent = (rect.xMinimum(), rect.yMinimum(), rect.xMaximum(), rect.yMaximum()) + + rasterize_grid_column( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + output_raster_path=output_path, + cell_size=self.cell_size_m, + extent=extent, + nodata=-9999.0, + area_name=area_name, + ) + + self.progressChanged.emit(100.0) + log_message(f"Rasterized grid column to {output_path}") + return output_path + + def _resolve_s2s_fields(self): + """Resolve ordered S2S fields configured for this indicator.""" + fields = self.attributes.get("s2s_fields", []) + if isinstance(fields, str): + fields = [token.strip() for token in fields.split(",") if token.strip()] + elif isinstance(fields, list): + fields = [str(token).strip() for token in fields if str(token).strip()] + else: + fields = [] + + unique_fields = [] + for field in fields: + if field not in unique_fields: + unique_fields.append(field) + + return unique_fields + + def _process_s2s_education_proxy(self, current_bbox: QgsGeometry, index: int, area_name: str) -> str: + """Process Education indicator using S2S urbanization population fields. + + Proxy rules: + - urban_pop = ghs_22_pop + ghs_23_pop + ghs_30_pop + - rural_pop = ghs_11_pop + ghs_12_pop + ghs_13_pop + - urban_share = urban_pop / ghs_total_pop + - likert score (1-5) from urban_share thresholds [0.2, 0.4, 0.6, 0.8] + + Notes: + - ghs_21_pop (suburban) is intentionally excluded. + - Cells with invalid/zero denominator are set to NULL. + """ + if not area_name: + raise ValueError("area_name is required for S2S education proxy processing.") + + if not os.path.exists(self.s2s_output_path): + raise ValueError(f"S2S output not found for Education proxy: {self.s2s_output_path}") + + required_fields = list(DEFAULT_S2S_EDUCATION_URBANIZATION_FIELDS) + missing_fields = [field for field in required_fields if field not in self.s2s_fields] + if missing_fields: + raise ValueError("Education S2S proxy requires fields " f"{required_fields}, but missing {missing_fields}.") + + source_layer = os.path.splitext(os.path.basename(self.s2s_output_path))[0] + temp_column_prefix = f"{self.layer_id}_s2s" + temp_columns = { + "ghs_11_pop": f"{temp_column_prefix}_11", + "ghs_12_pop": f"{temp_column_prefix}_12", + "ghs_13_pop": f"{temp_column_prefix}_13", + "ghs_22_pop": f"{temp_column_prefix}_22", + "ghs_23_pop": f"{temp_column_prefix}_23", + "ghs_30_pop": f"{temp_column_prefix}_30", + "ghs_total_pop": f"{temp_column_prefix}_total", + } + + # Clear target column once before first area, then write per-area values. + if not self._column_cleared: + clear_grid_column(self.gpkg_path, self.layer_id) + self._column_cleared = True + + self.progressChanged.emit(10.0) + + for source_field, temp_column in temp_columns.items(): + updated_count = write_joined_values_to_grid( + gpkg_path=self.gpkg_path, + column_name=temp_column, + source_gpkg=self.s2s_output_path, + source_layer=source_layer, + source_key_field="hex_id", + target_key_field="h3_index", + source_value_field=source_field, + area_name=area_name, + ) + if updated_count < 0: + raise RuntimeError(f"Failed joining S2S field '{source_field}' to grid for Education proxy.") + + self.progressChanged.emit(60.0) + + # Compute Likert score from urban share. + from osgeo import ogr + + ds = ogr.Open(self.gpkg_path, 1) + if not ds: + raise RuntimeError(f"Could not open GeoPackage for Education proxy update: {self.gpkg_path}") + + try: + q = lambda name: f'"{name.replace(" ", "_").replace("-", "_")[:63]}"' + + c11 = q(temp_columns["ghs_11_pop"]) + c12 = q(temp_columns["ghs_12_pop"]) + c13 = q(temp_columns["ghs_13_pop"]) + c22 = q(temp_columns["ghs_22_pop"]) + c23 = q(temp_columns["ghs_23_pop"]) + c30 = q(temp_columns["ghs_30_pop"]) + ctotal = q(temp_columns["ghs_total_pop"]) + target = q(self.layer_id) + + escaped_area_name = area_name.replace("'", "''") + where_area = f"area_name = '{escaped_area_name}'" + score_expr = ( + f"CASE " + f"WHEN COALESCE({ctotal}, 0) <= 0 THEN NULL " + f"WHEN ((COALESCE({c22},0)+COALESCE({c23},0)+COALESCE({c30},0)) / {ctotal}) < 0.2 THEN 1 " + f"WHEN ((COALESCE({c22},0)+COALESCE({c23},0)+COALESCE({c30},0)) / {ctotal}) < 0.4 THEN 2 " + f"WHEN ((COALESCE({c22},0)+COALESCE({c23},0)+COALESCE({c30},0)) / {ctotal}) < 0.6 THEN 3 " + f"WHEN ((COALESCE({c22},0)+COALESCE({c23},0)+COALESCE({c30},0)) / {ctotal}) < 0.8 THEN 4 " + f"ELSE 5 END" + ) + + sql = f"UPDATE study_area_grid SET {target} = {score_expr} WHERE {where_area}" # nosec B608 + ds.ExecuteSQL(sql, dialect="SQLite") + finally: + ds = None + + self.progressChanged.emit(85.0) + + # Rasterize from computed Education proxy column. + output_path = os.path.join( + self.workflow_directory, + f"{self.layer_id}_{index}.tif", + ) + + rect = current_bbox.boundingBox() + extent = (rect.xMinimum(), rect.yMinimum(), rect.xMaximum(), rect.yMaximum()) + + rasterize_grid_column( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + output_raster_path=output_path, + cell_size=self.cell_size_m, + extent=extent, + nodata=-9999.0, + area_name=area_name, + ) + + self.progressChanged.emit(100.0) + log_message( + f"Processed Education (S2S urbanization proxy) for area {area_name} into column {self.layer_id}", + tag="GeoE3", + level=Qgis.Info, + ) + return output_path + # Default implementation of the abstract method - not used in this workflow def _process_raster_for_area( self, @@ -121,6 +368,7 @@ def _process_raster_for_area( current_bbox: QgsGeometry, area_raster: str, index: int, + area_name: str = None, ): """ Executes the actual workflow logic for a single area using a raster. @@ -141,6 +389,7 @@ def _process_aggregate_for_area( clip_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: str = None, ): """ Executes the workflow, reporting progress through the feedback object and checking for cancellation. diff --git a/geest/core/workflows/polyline_per_cell_workflow.py b/geest/core/workflows/polyline_per_cell_workflow.py index 89cdc831..bb126503 100644 --- a/geest/core/workflows/polyline_per_cell_workflow.py +++ b/geest/core/workflows/polyline_per_cell_workflow.py @@ -2,9 +2,13 @@ """📦 Polyline Per Cell Workflow module. This module contains functionality for polyline per cell workflow. + +Supports grid-first mode where feature counts are written directly to +the study_area_grid column, then rasterized. """ import os +from typing import Optional from urllib.parse import unquote from qgis.core import ( @@ -16,9 +20,10 @@ ) from geest.core import JsonTreeItem -from geest.core.algorithms.features_per_cell_processor import ( - assign_values_to_grid, - select_grid_cells_and_count_features, +from geest.core.grid_column_utils import ( + clear_grid_column, + count_features_per_grid_cell, + rasterize_grid_column, ) from geest.utilities import log_message @@ -52,11 +57,9 @@ def __init__( item, cell_size_m, analysis_scale, feedback, context, working_directory ) # ⭐️ Item is a reference - whatever you change in this item will directly update the tree self.workflow_name = "use_polyline_per_cell" - layer_path = self.attributes.get("polyline_per_cell_shapefile", None) if layer_path: layer_path = unquote(layer_path) - if not layer_path: log_message( "Nothing found in polyline_per_cell_shapefile, trying polygline_per_cell_layer_source.", @@ -71,8 +74,12 @@ def __init__( level=Qgis.Warning, ) return False - self.features_layer = QgsVectorLayer(layer_path, "polyline_per_cell Layer", "ogr") + self.workflow_name = "polyline_per_cell" + # Grid-first mode: write results to grid columns first, then rasterize + self.use_grid_first = True + # Track if we've cleared the column (only do once, not per area) + self._column_cleared = False def _process_features_for_area( self, @@ -81,17 +88,20 @@ def _process_features_for_area( current_bbox: QgsGeometry, area_features: QgsVectorLayer, index: int, + area_name: Optional[str] = None, ) -> str: """ - Executes the actual workflow logic for a single area - Must be implemented by subclasses. + Executes the actual workflow logic for a single area. + + Supports grid-first mode where counts are written directly to study_area_grid. :current_area: Current polygon from our study area. :current_bbox: Bounding box of the above area. :area_features: A vector layer of features to analyse that includes only features in the study area. :index: Iteration / number of area being processed. + :area_name: Name of the area being processed. - :return: A raster layer file path if processing completes successfully, False if canceled or failed. + :return: A raster layer file path if processing completes successfully. """ area_features_count = area_features.featureCount() log_message( @@ -100,17 +110,35 @@ def _process_features_for_area( level=Qgis.Info, ) - # Step 1: Select grid cells that intersect with features - self.updateStatus(f"Counting intersections ({area_features_count} features)...") - output_path = os.path.join(self.workflow_directory, f"{self.layer_id}_grid_cells.gpkg") - area_grid = select_grid_cells_and_count_features(self.grid_layer, area_features, output_path, self.feedback) + if self.use_grid_first: + return self._process_grid_first( + current_bbox=current_bbox, + area_features=area_features, + index=index, + area_name=area_name, + ) + else: + return self._process_raster_first( + current_bbox=current_bbox, + area_features=area_features, + index=index, + ) - # Step 2: Assign values to grid cells - self.updateStatus("Assigning scores to grid cells...") - grid = assign_values_to_grid(area_grid, feedback=self.feedback) + def _process_raster_first( + self, + current_bbox: QgsGeometry, + area_features: QgsVectorLayer, + index: int, + ) -> str: + """Legacy raster-first processing using copied grid.""" + from geest.core.algorithms.features_per_cell_processor import ( + assign_values_to_grid, + select_grid_cells_and_count_features, + ) - # Step 3: Rasterize the grid layer using the assigned values - self.updateStatus("Rasterizing grid...") + output_path = os.path.join(self.workflow_directory, f"{self.layer_id}_grid_cells.gpkg") + area_grid = select_grid_cells_and_count_features(self.grid_layer, area_features, output_path, self.feedback) + grid = assign_values_to_grid(area_grid, self.feedback) raster_output = self._rasterize( grid, current_bbox, @@ -120,6 +148,56 @@ def _process_features_for_area( ) return raster_output + def _process_grid_first( + self, + current_bbox: QgsGeometry, + area_features: QgsVectorLayer, + index: int, + area_name: str, + ) -> str: + """Grid-first processing - writes directly to study_area_grid.""" + # Clear column once at the start (not per area) + if not self._column_cleared: + log_message(f"Clearing column {self.layer_id} before processing") + clear_grid_column(self.gpkg_path, self.layer_id) + self._column_cleared = True + + self.progressChanged.emit(10.0) + + # Count features and write to grid + log_message(f"Counting features for column {self.layer_id}") + count_features_per_grid_cell( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + features_layer=area_features, + feedback=self.feedback, + ) + + self.progressChanged.emit(50.0) + + # Rasterize from grid column + output_path = os.path.join( + self.workflow_directory, + f"{self.layer_id}_{index}.tif", + ) + + rect = current_bbox.boundingBox() + extent = (rect.xMinimum(), rect.yMinimum(), rect.xMaximum(), rect.yMaximum()) + + rasterize_grid_column( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + output_raster_path=output_path, + cell_size=self.cell_size_m, + extent=extent, + nodata=-9999.0, + area_name=area_name, + ) + + self.progressChanged.emit(100.0) + log_message(f"Rasterized grid column to {output_path}") + return output_path + # Default implementation of the abstract method - not used in this workflow def _process_raster_for_area( self, @@ -128,6 +206,7 @@ def _process_raster_for_area( current_bbox: QgsGeometry, area_raster: str, index: int, + area_name: str = None, ): """ Executes the actual workflow logic for a single area using a raster. @@ -148,6 +227,7 @@ def _process_aggregate_for_area( clip_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: str = None, ): """ Executes the workflow, reporting progress through the feedback object and checking for cancellation. diff --git a/geest/core/workflows/raster_reclassification_workflow.py b/geest/core/workflows/raster_reclassification_workflow.py index 4e67ffe2..4e87bc78 100644 --- a/geest/core/workflows/raster_reclassification_workflow.py +++ b/geest/core/workflows/raster_reclassification_workflow.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- """📦 Raster Reclassification Workflow module. - This module contains functionality for raster reclassification workflow. """ - import os from urllib.parse import unquote @@ -19,7 +17,8 @@ ) from geest.core import JsonTreeItem -from geest.core.constants import GDAL_OUTPUT_DATA_TYPE +from geest.core.constants import DEFAULT_S2S_ENV_HAZARD_FIELDS, DEFAULT_S2S_NTL_FIELD, GDAL_OUTPUT_DATA_TYPE +from geest.core.grid_column_utils import reclassify_grid_column_with_table, write_joined_values_to_grid from geest.utilities import log_message from .workflow_base import WorkflowBase @@ -52,16 +51,27 @@ def __init__( item, cell_size_m, analysis_scale, feedback, context, working_directory ) # ⭐️ Item is a reference - whatever you change in this item will directly update the tree self.workflow_name = "use_environmental_hazards" + self.s2s_output_path = self.attributes.get("s2s_output_path", "") + self.s2s_hazard_field = self._resolve_s2s_hazard_field() + self._use_s2s_grid_path = bool( + self.analysis_scale == "regional" and self.s2s_output_path and self.s2s_hazard_field + ) + self._configure_reclassification_rules() - if self.layer_id == "landslide": - self.range_boundaries = 2 # min and max values are included - else: - self.range_boundaries = 0 # default value for range boundaries + if self._use_s2s_grid_path: + self.features_layer = True + self.use_grid_first = True + self.raster_layer = None + log_message( + f"Using regional S2S grid path for environmental hazards ({self.layer_id}) field '{self.s2s_hazard_field}'.", + tag="GeoE3", + level=Qgis.Info, + ) + return layer_name = self.attributes.get("environmental_hazards_raster", None) if layer_name: layer_name = unquote(layer_name) - if not layer_name: log_message( "Invalid layer found in environmental_hazards_raster, trying environmental_hazards_layer_source.", @@ -76,114 +86,144 @@ def __init__( level=Qgis.Warning, ) return - self.raster_layer = QgsRasterLayer(layer_name, "Environmental Hazards Raster", "gdal") + def _resolve_s2s_hazard_field(self) -> str: + """Resolve and validate S2S hazard field for this indicator.""" + hazard_field = str(self.attributes.get("s2s_hazard_field", "") or "").strip() + fallback_field = DEFAULT_S2S_ENV_HAZARD_FIELDS.get(self.layer_id, "") + + if not hazard_field: + return fallback_field + + if hazard_field == DEFAULT_S2S_NTL_FIELD: + if fallback_field: + log_message( + f"S2S hazard field for {self.layer_id} was set to NTL field; using hazard default '{fallback_field}' instead.", + tag="GeoE3", + level=Qgis.Warning, + ) + return fallback_field + raise ValueError( + f"Invalid S2S hazard field for {self.layer_id}: '{hazard_field}'. Configure a hazard-specific field." + ) + + return hazard_field + + def _configure_reclassification_rules(self) -> None: + """Configure hazard-specific reclassification table and boundary mode.""" + if self.layer_id == "landslide": + self.range_boundaries = 2 # min and max values are included + else: + self.range_boundaries = 0 # default value for range boundaries + if self.layer_id == "fire": self.reclassification_rules = [ "-inf", 0, - 5.00, # new value = 5 + 5.00, 0, 1, - 4.00, # new value = 4 + 4.00, 1, 2, - 3.00, # new value = 3 + 3.00, 2, 5, - 2.00, # new value = 2 + 2.00, 5, 8, - 1.00, # new value = 1 + 1.00, 8, "inf", - 0, # new value = 0 + 0, ] elif self.layer_id == "flood": self.reclassification_rules = [ -1, 0, - 5.00, # new value = 5 + 5.00, 0, 180, - 4.00, # new value = 4 + 4.00, 180, 360, - 3.00, # new value = 3 + 3.00, 360, 540, - 2.00, # new value = 2 + 2.00, 540, 720, - 1.00, # new value = 1 + 1.00, 720, 900, - 0, # new value = 0 + 0, ] elif self.layer_id == "landslide": self.reclassification_rules = [ 0, 0, - 5.00, # new value = 5 + 5.00, 1, 1, - 4.00, # new value = 4 + 4.00, 2, 2, - 3.00, # new value = 3 + 3.00, 3, 3, - 2.00, # new value = 2 + 2.00, 4, 4, - 1.00, # new value = 1 + 1.00, 5, 5, - 0, # new value = 0 + 0, ] elif self.layer_id == "cyclone": self.reclassification_rules = [ 0, 0, - 5.00, # new value = 5 + 5.00, 0, 25, - 4.00, # new value = 4 + 4.00, 25, 50, - 3.00, # new value = 3 + 3.00, 50, 75, - 2.00, # new value = 2 + 2.00, 75, 100, - 1.00, # new value = 1 + 1.00, 100, "inf", - 0, # new value = 0 + 0, ] elif self.layer_id == "drought": self.reclassification_rules = [ 0, 0, - 5.00, # new value = 5 + 5.00, 0, 1, - 4.00, # new value = 4 + 4.00, 1, 2, - 3.00, # new value = 3 + 3.00, 2, 3, - 2.00, # new value = 2 + 2.00, 3, 4, - 1.00, # new value = 1 + 1.00, 4, 5, - 0, # new value = 0 + 0, ] + else: + raise ValueError(f"Unsupported environmental hazard layer id: {self.layer_id}") log_message( f"Reclassification Rules for {self.layer_id}: {self.reclassification_rules}", @@ -198,28 +238,25 @@ def _process_raster_for_area( current_bbox: QgsGeometry, area_raster: str, index: int, + area_name: str = None, ): """ Executes the actual workflow logic for a single area using a raster. - :current_area: Current polygon from our study area. :clip_area: Polygon to clip the raster to which is aligned to cell edges. :current_bbox: Bounding box of the above area. :area_raster: A raster layer of features to analyse that includes only bbox pixels in the study area. :index: Index of the current area. - :return: Path to the reclassified raster. """ del current_area # Unused in this analysis noqa F841 del clip_area # Unused in this analysis noqa F841 - # Apply the reclassification rules reclassified_raster = self._apply_reclassification( area_raster, index, bbox=current_bbox, ) - return reclassified_raster def _apply_reclassification( @@ -232,9 +269,7 @@ def _apply_reclassification( Apply the reclassification using the raster calculator and save the output. """ bbox = bbox.boundingBox() - reclassified_raster_path = os.path.join(self.workflow_directory, f"{self.layer_id}_reclassified_{index}.tif") - # Set up the reclassification using reclassifybytable params = { "INPUT_RASTER": input_raster, @@ -244,10 +279,8 @@ def _apply_reclassification( "OUTPUT": "TEMPORARY_OUTPUT", "PROGRESS": self.feedback, } - # Perform the reclassification using the raster calculator reclass = processing.run("native:reclassifybytable", params, feedback=QgsProcessingFeedback())["OUTPUT"] - clip_params = { "INPUT": reclass, "MASK": self.clip_areas_layer, @@ -258,14 +291,12 @@ def _apply_reclassification( "OUTPUT": reclassified_raster_path, "PROGRESS": self.feedback, } - processing.run("gdal:cliprasterbymasklayer", clip_params, feedback=QgsProcessingFeedback()) log_message( f"Reclassification for area {index} complete. Saved to {reclassified_raster_path}", tag="GeoE3", level=Qgis.Info, ) - return reclassified_raster_path # Not used in this workflow since we work with rasters @@ -276,20 +307,66 @@ def _process_features_for_area( current_bbox: QgsGeometry, area_features: QgsVectorLayer, index: int, + area_name: str = None, ) -> str: """ Executes the actual workflow logic for a single area Must be implemented by subclasses. - :current_area: Current polygon from our study area. :clip_area: Extended grid matched polygon for the study area. :current_bbox: Bounding box of the above area. :area_features: A vector layer of features to analyse that includes only features in the study area. :index: Iteration / number of area being processed. - :return: A raster layer file path if processing completes successfully, False if canceled or failed. """ - pass + _ = current_area + _ = clip_area + _ = area_features + + if not self._use_s2s_grid_path: + return None + + if not area_name: + raise ValueError("area_name is required for regional S2S environmental hazards processing.") + + source_layer = os.path.splitext(os.path.basename(self.s2s_output_path))[0] + updated_count = write_joined_values_to_grid( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + source_gpkg=self.s2s_output_path, + source_layer=source_layer, + source_key_field="hex_id", + target_key_field="h3_index", + source_value_field=self.s2s_hazard_field, + area_name=area_name, + ) + + if updated_count < 0: + raise RuntimeError("Failed to write S2S environmental hazards values to study_area_grid.") + + mapped_count = reclassify_grid_column_with_table( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + reclassification_table=self.reclassification_rules, + area_name=area_name, + range_boundaries=self.range_boundaries, + ) + if mapped_count < 0: + raise RuntimeError("Failed to map S2S environmental hazards values to Likert scale.") + + log_message( + f"Wrote {updated_count} regional S2S environmental hazards values and mapped {mapped_count} cells " + f"to Likert scale in grid column {self.layer_id}", + tag="GeoE3", + level=Qgis.Info, + ) + + return self._rasterize_grid_column( + column_name=self.layer_id, + bbox=current_bbox, + area_name=area_name, + index=index, + ) def _process_aggregate_for_area( self, @@ -297,15 +374,14 @@ def _process_aggregate_for_area( clip_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: str = None, ): """ Executes the actual workflow logic for a single area using an aggregate. - :current_area: Current polygon from our study area. :clip_area: Extended grid matched polygon for the study area. :current_bbox: Bounding box of the above area. :index: Index of the current area. - :return: Path to the reclassified raster. """ pass diff --git a/geest/core/workflows/safety_polygon_workflow.py b/geest/core/workflows/safety_polygon_workflow.py index 871780e1..c2fd7edd 100644 --- a/geest/core/workflows/safety_polygon_workflow.py +++ b/geest/core/workflows/safety_polygon_workflow.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- """📦 Safety Polygon Workflow module. - This module contains functionality for safety polygon workflow. """ - from urllib.parse import unquote from qgis.core import ( @@ -51,7 +49,6 @@ def __init__( ) # ⭐️ Item is a reference - whatever you change in this item will directly update the tree self.workflow_name = "use_classify_safety_polygon_into_classes" layer_path = unquote(self.attributes.get("classify_safety_polygon_into_classes_shapefile", "")) - if not layer_path: log_message( "Invalid layer found in classify_safety_polygon_into_classes_shapefile, trying classify_safety_polygon_into_classes_layer_source.", @@ -66,9 +63,7 @@ def __init__( level=Qgis.Warning, ) return False - self.features_layer = QgsVectorLayer(layer_path, "features_layer", "ogr") - self.selected_field = self.attributes.get("classify_safety_polygon_into_classes_selected_field", "") # This is a dict with keys being unique values from the selected field # and values from the aggregation dialog configuration table @@ -83,16 +78,15 @@ def _process_features_for_area( current_bbox: QgsGeometry, area_features: QgsVectorLayer, index: int, + area_name: str = None, ) -> str: """ Executes the actual workflow logic for a single area Must be implemented by subclasses. - :current_area: Current polygon from our study area. :current_bbox: Bounding box of the above area. :area_features: A vector layer of features to analyse that includes only features in the study area. :index: Iteration / number of area being processed. - :return: A raster layer file path if processing completes successfully, False if canceled or failed. """ area_features_count = area_features.featureCount() @@ -103,7 +97,6 @@ def _process_features_for_area( ) # Step 1: Assign reclassification values based on perceived safety reclassified_layer = self._assign_reclassification_to_safety(area_features) - # Step 2: Rasterize the safety data raster_output = self._rasterize( reclassified_layer, @@ -122,7 +115,6 @@ def _assign_reclassification_to_safety(self, layer: QgsVectorLayer) -> QgsVector if layer.fields().indexFromName("value") == -1: layer.dataProvider().addAttributes([QgsField("value", QVariant.Int)]) layer.updateFields() - feature_count = layer.featureCount() counter = 0 for feature in layer.getFeatures(): @@ -159,16 +151,15 @@ def _process_raster_for_area( current_bbox: QgsGeometry, area_raster: str, index: int, + area_name: str = None, ): """ Executes the actual workflow logic for a single area using a raster. - :current_area: Current polygon from our study area. :clip_area: Polygon to clip the raster to which is aligned to cell edges. :current_bbox: Bounding box of the above area. :area_raster: A raster layer of features to analyse that includes only bbox pixels in the study area. :index: Index of the current area. - :return: Path to the reclassified raster. """ pass @@ -179,6 +170,7 @@ def _process_aggregate_for_area( clip_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: str = None, ): """ Executes the workflow, reporting progress through the feedback object and checking for cancellation. diff --git a/geest/core/workflows/safety_raster_workflow.py b/geest/core/workflows/safety_raster_workflow.py index 5b6475ed..21fdf673 100644 --- a/geest/core/workflows/safety_raster_workflow.py +++ b/geest/core/workflows/safety_raster_workflow.py @@ -1,10 +1,9 @@ # -*- coding: utf-8 -*- """📦 Safety Raster Workflow module. - This module contains functionality for safety raster workflow. """ - import os +from typing import Optional from urllib.parse import unquote import numpy as np @@ -20,6 +19,7 @@ ) from geest.core import JsonTreeItem +from geest.core.grid_column_utils import write_joined_values_to_grid, write_spatial_join_to_grid from geest.core.jenks import calculate_goodness_of_variance_fit, jenks_natural_breaks from geest.utilities import log_message @@ -55,8 +55,54 @@ def __init__( item, cell_size_m, analysis_scale, feedback, context, working_directory ) # ⭐️ Item is a reference - whatever you change in this item will directly update the tree self.workflow_name = "use_nighttime_lights" - layer_name = unquote(self.attributes.get("nighttime_lights_raster", None)) + self.s2s_output_path = self.attributes.get("s2s_output_path", "") + self.s2s_ntl_field = self.attributes.get("s2s_ntl_field", "") + self.vector_source_path = unquote(self.attributes.get("nighttime_lights_vector", "")) + self.vector_value_field = self.attributes.get("nighttime_lights_selected_field", "") + self._use_s2s_grid_path = bool( + self.analysis_scale == "regional" and self.s2s_output_path and self.s2s_ntl_field + ) + self._use_vector_path = bool(self.vector_source_path and self.vector_value_field) + + if self._use_s2s_grid_path: + self.features_layer = True + self.use_grid_first = True + self.raster_layer = None + log_message( + "Using regional S2S grid path for nighttime lights (direct raw value write).", + tag="GeoE3", + level=Qgis.Info, + ) + return + + if self._use_vector_path: + self.features_layer = QgsVectorLayer(self.vector_source_path, "Nighttime Lights Vector", "ogr") + if not self.features_layer.isValid(): + log_message( + f"Invalid nighttime lights vector source: {self.vector_source_path}. Falling back to raster path.", + tag="GeoE3", + level=Qgis.Warning, + ) + self.features_layer = None + else: + self.use_grid_first = True + self.raster_layer = None + log_message( + f"Using vector nighttime lights path with field '{self.vector_value_field}'.", + tag="GeoE3", + level=Qgis.Info, + ) + return + + if self.analysis_scale == "regional": + log_message( + "Regional nighttime lights S2S data not configured; falling back to raster input path.", + tag="GeoE3", + level=Qgis.Warning, + ) + + layer_name = unquote(self.attributes.get("nighttime_lights_raster", None)) if not layer_name: log_message( "Invalid raster found in nighttime_lights_raster, trying nighttime_lights_layer_source.", @@ -80,22 +126,19 @@ def _process_raster_for_area( current_bbox: QgsGeometry, area_raster: str, index: int, + area_name: str = None, ): """ Executes the actual workflow logic for a single area using a raster. - :current_area: Current polygon from our study area. :clip_area: Polygon to clip the raster to which is aligned to cell edges. :current_bbox: Bounding box of the above area. :area_raster: A raster layer of features to analyse that includes only bbox pixels in the study area. :index: Index of the current area. - :return: Path to the reclassified raster. """ _ = current_area # Unused in this analysis - max_val, median, percentile_75, valid_data = self.calculate_raster_stats(area_raster) - # Check if we got valid statistics if valid_data is None or len(valid_data) == 0: log_message( @@ -104,7 +147,6 @@ def _process_raster_for_area( level=1, ) return None - # Dynamically build the reclassification table using Jenks Natural Breaks reclass_table = self._build_reclassification_table(max_val, median, valid_data) log_message( @@ -112,7 +154,6 @@ def _process_raster_for_area( tag="GeoE3", level=0, ) - # Apply the reclassification rules reclassified_raster = self._apply_reclassification( area_raster, @@ -122,6 +163,134 @@ def _process_raster_for_area( ) return reclassified_raster + def _process_features_for_area( + self, + current_area: QgsGeometry, + clip_area: QgsGeometry, + current_bbox: QgsGeometry, + area_features: QgsVectorLayer, + index: int, + area_name: str = None, + ) -> Optional[str]: + """Process S2S regional nighttime lights by writing raw values to the H3 grid.""" + _ = current_area + _ = clip_area + _ = area_features + + if not self._use_s2s_grid_path and not self._use_vector_path: + return None + + if not area_name: + raise ValueError("area_name is required for regional S2S nighttime lights processing.") + + if self._use_s2s_grid_path: + source_layer = os.path.splitext(os.path.basename(self.s2s_output_path))[0] + updated_count = write_joined_values_to_grid( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + source_gpkg=self.s2s_output_path, + source_layer=source_layer, + source_key_field="hex_id", + target_key_field="h3_index", + source_value_field=self.s2s_ntl_field, + area_name=area_name, + ) + + if updated_count < 0: + raise RuntimeError("Failed to write S2S nighttime lights values to study_area_grid.") + + mapped_count = self._apply_s2s_ntl_likert_mapping(area_name=area_name) + if mapped_count < 0: + raise RuntimeError("Failed to map S2S nighttime lights values to Likert scale.") + + log_message( + f"Wrote {updated_count} regional S2S nighttime lights values and mapped {mapped_count} cells " + f"to Likert scale in grid column {self.layer_id}", + tag="GeoE3", + level=Qgis.Info, + ) + else: + source_path = area_features.source() + source_layer = os.path.splitext(os.path.basename(source_path))[0] + updated_count = write_spatial_join_to_grid( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + features_gpkg=source_path, + features_layer=source_layer, + score_expression=self.vector_value_field, + area_name=area_name, + aggregation_method="MAX", + save_buffers=False, + workflow_directory=self.workflow_directory, + ) + + if updated_count < 0: + raise RuntimeError("Failed to write vector nighttime lights values to study_area_grid.") + + log_message( + f"Wrote {updated_count} vector nighttime lights values to grid column {self.layer_id}", + tag="GeoE3", + level=Qgis.Info, + ) + + return self._rasterize_grid_column( + column_name=self.layer_id, + bbox=current_bbox, + area_name=area_name, + index=index, + ) + + def _apply_s2s_ntl_likert_mapping(self, area_name: str) -> int: + """Map joined S2S NTL values to the same Likert scale as raster mode.""" + from geest.core.grid_column_utils import get_grid_column_values, reclassify_grid_column_with_table + + classification_mode = self.attributes.get("ntl_classification_mode", "jenks") + if classification_mode == "binary": + reclass_table = ["-inf", "0.0", "0", "0.0", "inf", "5"] + return reclassify_grid_column_with_table( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + reclassification_table=reclass_table, + area_name=area_name, + range_boundaries=0, + ) + + values = get_grid_column_values(self.gpkg_path, self.layer_id, area_name=area_name) + if not values: + return 0 + + valid_data = np.asarray(values, dtype=float) + breaks = jenks_natural_breaks(valid_data, n_classes=6) + _ = calculate_goodness_of_variance_fit(valid_data, breaks) + + reclass_table = [ + str(0.0), + str(breaks[0]), + "0", + str(breaks[0]), + str(breaks[1]), + "1", + str(breaks[1]), + str(breaks[2]), + "2", + str(breaks[2]), + str(breaks[3]), + "3", + str(breaks[3]), + str(breaks[4]), + "4", + str(breaks[4]), + str(breaks[5]), + "5", + ] + return reclassify_grid_column_with_table( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + reclassification_table=reclass_table, + area_name=area_name, + range_boundaries=0, + ) + def _apply_reclassification( self, input_raster: QgsRasterLayer, @@ -133,9 +302,7 @@ def _apply_reclassification( Apply the reclassification using the raster calculator and save the output. """ bbox = bbox.boundingBox() - reclassified_raster = os.path.join(self.workflow_directory, f"{self.layer_id}_reclassified_{index}.tif") - # Set up the reclassification using reclassifybytable params = { "INPUT_RASTER": input_raster, @@ -147,46 +314,38 @@ def _apply_reclassification( "OUTPUT": reclassified_raster, "PROGRESS": self.feedback, } - # Perform the reclassification using the raster calculator processing.run( "native:reclassifybytable", # noqa F841 params, # noqa F841 feedback=QgsProcessingFeedback(), # noqa F841 )["OUTPUT"] - log_message( f"Reclassification for area {index} complete. Saved to {reclassified_raster}", tag="GeoE3", level=Qgis.Info, ) - return reclassified_raster def calculate_raster_stats(self, raster_path): """ Calculate statistics from a QGIS raster layer using NumPy. - Returns: Tuple of (max_value, median, percentile_75, valid_data) Returns (None, None, None, None) if raster cannot be read """ raster_layer = QgsRasterLayer(raster_path, "Input Raster") - # Check if the raster layer loaded successfully if not raster_layer.isValid(): log_message("Raster layer failed to load", tag="GeoE3", level=1) return None, None, None, None - provider = raster_layer.dataProvider() extent = raster_layer.extent() width = raster_layer.width() height = raster_layer.height() - # Fetch the raster data for band 1 block = provider.block(1, extent, width, height) byte_array = block.data() # This returns a QByteArray - # Determine the correct dtype based on the provider's data type data_type = provider.dataType(1) dtype = None @@ -202,24 +361,19 @@ def calculate_raster_stats(self, raster_path): dtype = np.uint32 elif data_type == 1: # Byte dtype = np.uint8 - if dtype is None: log_message("Unsupported data type", tag="GeoE3", level=1) return None, None, None, None - # Convert QByteArray to a numpy array with the correct dtype raster_array = np.frombuffer(byte_array, dtype=dtype).reshape((height, width)) - # Filter out NoData values no_data_value = provider.sourceNoDataValue(1) valid_data = raster_array[raster_array != no_data_value] - if valid_data.size > 0: # Compute statistics max_value = np.max(valid_data).astype(dtype) median = np.median(valid_data).astype(dtype) percentile_75 = np.percentile(valid_data, 75).astype(dtype) - return max_value, median, percentile_75, valid_data else: # Handle case with no valid data @@ -239,7 +393,6 @@ def _build_binary_table(self, max_val: float) -> list: Args: max_val: Maximum value in the raster data - Returns: Reclassification table as list of strings Format: [min1, max1, class1, min2, max2, class2] @@ -260,11 +413,9 @@ def _build_reclassification_table(self, max_val: float, median: float, valid_dat max_val: Maximum value in the raster median: Median value in the raster valid_data: Array of all valid (non-NoData) raster values - Returns: Reclassification table as list of [min, max, class, min, max, class, ...] formatted as strings for QGIS native:reclassifybytable algorithm - Raises: ValueError: If Jenks Natural Breaks cannot compute valid classification breaks """ @@ -281,14 +432,12 @@ def _build_reclassification_table(self, max_val: float, median: float, valid_dat # Jenks Natural Breaks n_classes = 6 - log_message( f"📊 Computing Jenks Natural Breaks classification (max={max_val:.6f}, " f"median={median:.6f}, n={len(valid_data)})", tag="GeoE3", level=0, ) - try: # Returns: [break₁, break₂, break₃, break₄, break₅, max_value] breaks = jenks_natural_breaks(valid_data, n_classes=n_classes) @@ -297,20 +446,16 @@ def _build_reclassification_table(self, max_val: float, median: float, valid_dat # Build QGIS reclassification table format # Format: [min₁, max₁, class₁, min₂, max₂, class₂, ...] reclass_table = [] - # Class 0: From 0 to first break reclass_table.extend([0.0, breaks[0], 0]) - # Classes 1-5: Between consecutive breaks for i in range(len(breaks) - 1): class_num = i + 1 min_val = breaks[i] max_val_class = breaks[i + 1] reclass_table.extend([min_val, max_val_class, class_num]) - # Convert all values to strings for QGIS processing reclass_table = list(map(str, reclass_table)) - log_message( f"✅ Jenks Natural Breaks computed:\n" f" Class 0 (No Access): 0.000 - {breaks[0]:.3f}\n" @@ -323,9 +468,7 @@ def _build_reclassification_table(self, max_val: float, median: float, valid_dat tag="GeoE3", level=0, ) - return reclass_table - except Exception as e: # Fail workflow with clear error message unique_count = len(np.unique(valid_data)) @@ -342,32 +485,12 @@ def _build_reclassification_table(self, max_val: float, median: float, valid_dat log_message(error_msg, tag="GeoE3", level=2) # Critical raise ValueError(error_msg) from e - # Not used in this workflow since we work with rasters - def _process_features_for_area( - self, - current_area: QgsGeometry, - current_bbox: QgsGeometry, - area_features: QgsVectorLayer, - index: int, - ) -> str: - """ - Executes the actual workflow logic for a single area - Must be implemented by subclasses. - - :current_area: Current polygon from our study area. - :current_bbox: Bounding box of the above area. - :area_features: A vector layer of features to analyse that includes only features in the study area. - :index: Iteration / number of area being processed. - - :return: A raster layer file path if processing completes successfully, False if canceled or failed. - """ - pass - def _process_aggregate_for_area( self, current_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: str = None, ): """ Executes the workflow, reporting progress through the feedback object and checking for cancellation. diff --git a/geest/core/workflows/single_point_buffer_workflow.py b/geest/core/workflows/single_point_buffer_workflow.py index 439e138c..d6595bd1 100644 --- a/geest/core/workflows/single_point_buffer_workflow.py +++ b/geest/core/workflows/single_point_buffer_workflow.py @@ -2,9 +2,13 @@ """📦 Single Point Buffer Workflow module. This module contains functionality for single point buffer workflow. + +Supports grid-first mode where buffer scores are written directly to +the study_area_grid column, then rasterized. """ import os +from typing import Optional from urllib.parse import unquote from qgis import processing @@ -19,6 +23,11 @@ from qgis.PyQt.QtCore import QVariant from geest.core import JsonTreeItem +from geest.core.grid_column_utils import ( + clear_grid_column, + rasterize_grid_column, + write_buffer_values_to_grid, +) from geest.core.workflows.mappings import MAPPING_REGISTRY from geest.utilities import log_message @@ -52,7 +61,6 @@ def __init__( item, cell_size_m, analysis_scale, feedback, context, working_directory ) # ⭐️ Item is a reference - whatever you change in this item will directly update the tree self.workflow_name = "use_single_buffer_point" - layer_source = self.attributes.get("single_buffer_point_shapefile", None) if layer_source is not None: layer_source = unquote(layer_source) @@ -81,39 +89,69 @@ def __init__( config = mapping.get(analysis_scale, mapping.get("national")) if mapping else None mapped_distance = config.get("buffer_distance") if config else None self.mapped_scores = config.get("scores", {}) if config else {} - # Load scoring method and percentage scores for Regional scale self.scoring_method = config.get("scoring_method", "") if config else "" self.percentage_scores = config.get("percentage_scores", {}) if config else {} - default_buffer_distance = int(self.attributes.get("default_single_buffer_distance", 0)) if mapped_distance: default_buffer_distance = int(mapped_distance) - buffer_distance = self.attributes.get("single_buffer_point_layer_distance", default_buffer_distance) self.buffer_distance = int(buffer_distance) if buffer_distance else int(default_buffer_distance) + self.workflow_name = "single_point_buffer" + # Grid-first mode: write results to grid columns first, then rasterize + self.use_grid_first = True + # Track if we've cleared the column (only do once, not per area) + self._column_cleared = False def _process_features_for_area( self, - current_area: "QgsGeometry", - clip_area: "QgsGeometry", - current_bbox: "QgsGeometry", - area_features: "QgsVectorLayer", + current_area: QgsGeometry, + clip_area: QgsGeometry, + current_bbox: QgsGeometry, + area_features: QgsVectorLayer, index: int, + area_name: Optional[str] = None, ) -> str: """ - Executes the actual workflow logic for a single area - Must be implemented by subclasses. + Executes the actual workflow logic for a single area. + + Supports grid-first mode where buffer scores are written directly to study_area_grid. :current_area: Current polygon from our study area. :current_bbox: Bounding box of the above area. :area_features: A vector layer of features to analyse that includes only features in the study area. :index: Iteration / number of area being processed. + :area_name: Name of the area being processed. - :return: A raster layer file path if processing completes successfully, False if canceled or failed. + :return: A raster layer file path if processing completes successfully. """ - log_message(f"{self.workflow_name} Processing Started") + log_message(f"{self.workflow_name} Processing Started") + + if self.use_grid_first: + return self._process_grid_first( + current_bbox=current_bbox, + area_features=area_features, + index=index, + area_name=area_name, + ) + else: + return self._process_raster_first( + current_area=current_area, + clip_area=clip_area, + current_bbox=current_bbox, + area_features=area_features, + index=index, + ) + def _process_raster_first( + self, + current_area: QgsGeometry, + clip_area: QgsGeometry, + current_bbox: QgsGeometry, + area_features: QgsVectorLayer, + index: int, + ) -> str: + """Legacy raster-first processing.""" # Check if we should use percentage-based scoring (Regional scale) if self.scoring_method == "percentage_intersection" and self.percentage_scores: return self._process_with_percentage_scoring( @@ -123,26 +161,86 @@ def _process_features_for_area( area_features=area_features, index=index, ) - # Original binary scoring approach for National/Local scale # Step 1: Buffer the selected features buffered_layer = self._buffer_features(area_features, f"{self.layer_id}_buffered_{index}") - # Step 2: Assign values to the buffered polygons scored_layer = self._assign_scores(buffered_layer) - # Step 3: Rasterize the scored buffer layer raster_output = self._rasterize(scored_layer, current_bbox, index) - return raster_output + def _process_grid_first( + self, + current_bbox: QgsGeometry, + area_features: QgsVectorLayer, + index: int, + area_name: str, + ) -> str: + """Grid-first processing - writes buffer scores directly to study_area_grid.""" + # Clear column once at the start (not per area) + if not self._column_cleared: + log_message(f"Clearing column {self.layer_id} before processing") + clear_grid_column(self.gpkg_path, self.layer_id) + self._column_cleared = True + + self.progressChanged.emit(10.0) + + # Step 1: Buffer the selected features + log_message(f"Creating buffer with distance {self.buffer_distance}m") + buffered_layer = self._buffer_features(area_features, f"{self.layer_id}_buffered_{index}") + + self.progressChanged.emit(30.0) + + # Step 2: Assign values to the buffered polygons + scored_layer = self._assign_scores(buffered_layer) + + self.progressChanged.emit(40.0) + + # Step 3: Write buffer scores to grid cells + log_message(f"Writing buffer scores to grid column {self.layer_id}") + write_buffer_values_to_grid( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + buffer_layer=scored_layer, + value_field="value", + aggregation_method="MAX", + feedback=self.feedback, + ) + + self.progressChanged.emit(70.0) + + # Step 4: Rasterize from grid column + output_path = os.path.join( + self.workflow_directory, + f"{self.layer_id}_{index}.tif", + ) + + rect = current_bbox.boundingBox() + extent = (rect.xMinimum(), rect.yMinimum(), rect.xMaximum(), rect.yMaximum()) + + rasterize_grid_column( + gpkg_path=self.gpkg_path, + column_name=self.layer_id, + output_raster_path=output_path, + cell_size=self.cell_size_m, + extent=extent, + nodata=-9999.0, + area_name=area_name, + ) + + self.progressChanged.emit(100.0) + log_message(f"Rasterized grid column to {output_path}") + return output_path + def _process_with_percentage_scoring( self, - current_area: "QgsGeometry", - clip_area: "QgsGeometry", - current_bbox: "QgsGeometry", - area_features: "QgsVectorLayer", + current_area: QgsGeometry, + clip_area: QgsGeometry, + current_bbox: QgsGeometry, + area_features: QgsVectorLayer, index: int, + area_name: str = None, ) -> str: """Process using percentage-based scoring (Regional scale). @@ -165,12 +263,10 @@ def _process_with_percentage_scoring( f"Single Point Buffer: Using percentage scoring for Regional scale (buffer: {self.buffer_distance}m)", level=Qgis.Info, ) - # Step 1: Create buffer around points buffer_output = os.path.join(self.workflow_directory, f"simple_buffer_{index}.gpkg") if os.path.exists(buffer_output): os.remove(buffer_output) - buffer_params = { "INPUT": area_features, "DISTANCE": self.buffer_distance, @@ -178,14 +274,12 @@ def _process_with_percentage_scoring( } result = processing.run("native:buffer", buffer_params) buffered_layer_path = result["OUTPUT"] - if not buffered_layer_path or not os.path.exists(buffered_layer_path): log_message( f"Failed to create buffer for area {index}", level=Qgis.Warning, ) return False - buffered_layer = QgsVectorLayer(buffered_layer_path, "buffered", "ogr") if not buffered_layer.isValid(): log_message( @@ -193,7 +287,6 @@ def _process_with_percentage_scoring( level=Qgis.Warning, ) return False - # Step 2: Get grid cells for current area area_grid = subset_vector_layer( self.workflow_directory, @@ -207,20 +300,17 @@ def _process_with_percentage_scoring( level=Qgis.Warning, ) return False - # Step 3: Score grid cells based on percentage intersection scored_grid = self._score_grid_for_percentage( grid_layer=area_grid, buffered_layer=buffered_layer, ) - if scored_grid is False: log_message( "No scored grid cells were created.", level=Qgis.Warning, ) return False - # Step 4: Rasterize raster_output = self._rasterize( input_layer=scored_grid, @@ -228,14 +318,13 @@ def _process_with_percentage_scoring( index=index, value_field="value", ) - return raster_output def _score_grid_for_percentage( self, - grid_layer: "QgsVectorLayer", - buffered_layer: "QgsVectorLayer", - ) -> "QgsVectorLayer": + grid_layer: QgsVectorLayer, + buffered_layer: QgsVectorLayer, + ) -> QgsVectorLayer: """Score grid cells based on percentage intersection with buffered features. For Regional scale: calculates what percentage of each grid cell @@ -249,44 +338,34 @@ def _score_grid_for_percentage( The grid layer with "value" field containing the assigned scores. """ log_message("Scoring grid cells based on percentage intersection") - field_names = [field.name() for field in grid_layer.fields()] if "value" not in field_names: grid_layer.dataProvider().addAttributes([QgsField("value", QVariant.Int)]) grid_layer.updateFields() - grid_layer.startEditing() for grid_feature in grid_layer.getFeatures(): grid_geom = grid_feature.geometry() if grid_geom.isNull(): continue - grid_area = grid_geom.area() if grid_area == 0: continue - - max_score = 0 max_overlap_percent = 0 - for buffered_feature in buffered_layer.getFeatures(): buffered_geom = buffered_feature.geometry() if buffered_geom.isNull(): continue - intersection = grid_geom.intersection(buffered_geom) if intersection.isNull() or intersection.area() == 0: continue - # Calculate % of buffer within hexagon buffer_area = buffered_geom.area() if buffer_area > 0: overlap_percent = (intersection.area() / buffer_area) * 100 else: overlap_percent = 0 - if overlap_percent > max_overlap_percent: max_overlap_percent = overlap_percent - # Calculate score based on final max_overlap_percent after all buffers checked sorted_items = sorted(self.percentage_scores.items()) max_score = 0 @@ -304,15 +383,11 @@ def _score_grid_for_percentage( prev_pct = sorted_items[i - 1][0] if prev_pct < max_overlap_percent <= min_pct: max_score = score - grid_feature.setAttribute("value", max_score) grid_layer.updateFeature(grid_feature) - grid_layer.commitChanges() return grid_layer - return raster_output - def _buffer_features(self, layer: QgsVectorLayer, output_name: str) -> QgsVectorLayer: """ Buffer the input features by the buffer_distance km. @@ -335,7 +410,6 @@ def _buffer_features(self, layer: QgsVectorLayer, output_name: str) -> QgsVector "OUTPUT": output_path, }, )["OUTPUT"] - buffered_layer = QgsVectorLayer(output_path, output_name, "ogr") return buffered_layer @@ -349,21 +423,17 @@ def _assign_scores(self, layer: QgsVectorLayer) -> QgsVectorLayer: Returns: QgsVectorLayer: A new layer with a "value" field containing the assigned scores. """ - log_message(f"Assigning scores to {layer.name()}") # Create a new field in the layer for the scores layer.startEditing() layer.dataProvider().addAttributes([QgsField("value", QVariant.Int)]) layer.updateFields() - # Assign scores to the buffered polygons score = self.mapped_scores.get("intersects", 5) for feature in layer.getFeatures(): feature.setAttribute("value", score) layer.updateFeature(feature) - layer.commitChanges() - return layer # Default implementation of the abstract method - not used in this workflow @@ -374,6 +444,7 @@ def _process_raster_for_area( current_bbox: QgsGeometry, area_raster: str, index: int, + area_name: str = None, ): """ Executes the actual workflow logic for a single area using a raster. @@ -394,6 +465,7 @@ def _process_aggregate_for_area( clip_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: str = None, ): """ Executes the workflow, reporting progress through the feedback object and checking for cancellation. diff --git a/geest/core/workflows/street_lights_buffer_workflow.py b/geest/core/workflows/street_lights_buffer_workflow.py index a0927702..bd27e6f5 100644 --- a/geest/core/workflows/street_lights_buffer_workflow.py +++ b/geest/core/workflows/street_lights_buffer_workflow.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- """📦 Street Lights Buffer Workflow module. - This module contains functionality for street lights buffer workflow. """ - import os from urllib.parse import unquote @@ -42,20 +40,14 @@ def __init__( ): """ Initialize the workflow with attributes and feedback. - :param item: JsonTreeItem representing the analysis, dimension, or factor to process. - :param cell_size_m: Cell size in meters. - :param analysis_scale: Scale of the analysis, e.g., 'local', 'national'. - :param feedback: QgsFeedback object for progress reporting and cancellation's. - :param context: QgsProcessingContext object for processing. This can be used to pass objects to the thread. e.g. the QgsProject Instance. - :param working_directory: Folder containing study_area.gpkg where the outputs will be placed. If not set, the value will be taken from QSettings. @@ -64,11 +56,9 @@ def __init__( # this item will directly update the tree super().__init__(item, cell_size_m, analysis_scale, feedback, context, working_directory) self.workflow_name = "use_street_lights" - layer_path = self.attributes.get("street_lights_shapefile", None) if layer_path: layer_path = unquote(layer_path) - if not layer_path: log_message( "Invalid raster found in street_lights_shapefile, trying street_lights_point_layer_source.", @@ -87,7 +77,6 @@ def __init__( error += f"Layer Source: {self.street_lights_layer_source} " self.attributes["error"] = error raise Exception(error) - self.features_layer = QgsVectorLayer(layer_path, "points", "ogr") if not self.features_layer.isValid(): log_message("street lights source file not valid", level=Qgis.Critical) @@ -96,7 +85,6 @@ def __init__( error += f"Layer Source: {self.street_lights_layer_source} " self.attributes["error"] = error raise Exception(error) - factor_id = None if item.isIndicator() and item.parentItem: factor_id = item.parentItem.attribute("id", None) @@ -108,7 +96,6 @@ def __init__( config = mapping.get(analysis_scale, mapping.get("national")) if not config: raise Exception("Streetlights mapping config not found.") - self.buffer_distance = int(config.get("buffer_distance", 0)) self.scoring_method = config.get("scoring_method", "") self.scores = config.get("scores", {}) @@ -123,29 +110,25 @@ def _process_features_for_area( current_bbox: QgsGeometry, area_features: QgsVectorLayer, index: int, + area_name: str = None, ) -> str: """ Executes the actual workflow logic for a single area Must be implemented by subclasses. - :current_area: Current polygon from our study area. :current_bbox: Bounding box of the above area. :area_features: A vector layer of features to analyse that includes only features in the study area. :index: Iteration / number of area being processed. - :return: A raster layer file path if processing completes successfully, False if canceled or failed. """ log_message(f"{self.workflow_name} Processing Started") - # Step 1: Buffer the selected features buffered_layer = self._buffer_features(area_features, f"{self.layer_id}_buffered_{index}") # Step 2: Select grid cells that intersect with features output_path = os.path.join(self.workflow_directory, f"{self.layer_id}_grid_cells.gpkg") area_grid = select_grid_cells_and_count_features(self.grid_layer, area_features, output_path, self.feedback) - # Step 3: Assign scores to the grid layer grid_layer = self._score_grid(area_grid, buffered_layer) - # Step 4: Rasterize the grid layer using the assigned scores raster_output = self._rasterize( grid_layer, @@ -154,17 +137,14 @@ def _process_features_for_area( value_field="score", default_value=0, ) - return raster_output def _buffer_features(self, layer: QgsVectorLayer, output_name: str) -> QgsVectorLayer: """ Buffer the input features by the buffer_distance km. - Args: layer (QgsVectorLayer): The input feature layer. output_name (str): A name for the output buffered layer. - Returns: QgsVectorLayer: The buffered features layer. """ @@ -179,7 +159,6 @@ def _buffer_features(self, layer: QgsVectorLayer, output_name: str) -> QgsVector "OUTPUT": output_path, }, )["OUTPUT"] - buffered_layer = QgsVectorLayer(output_path, output_name, "ogr") return buffered_layer @@ -191,16 +170,15 @@ def _process_raster_for_area( current_bbox: QgsGeometry, area_raster: str, index: int, + area_name: str = None, ): """ Executes the actual workflow logic for a single area using a raster. - :current_area: Current polygon from our study area. :clip_area: Polygon to clip the raster to which is aligned to cell edges. :current_bbox: Bounding box of the above area. :area_raster: A raster layer of features to analyse that includes only bbox pixels in the study area. :index: Index of the current area. - :return: Path to the reclassified raster. """ pass @@ -211,6 +189,7 @@ def _process_aggregate_for_area( clip_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: str = None, ): """ Executes the workflow, reporting progress through the feedback object and checking for cancellation. @@ -220,12 +199,10 @@ def _process_aggregate_for_area( def _score_grid(self, grid_layer: QgsVectorLayer, buffered_layer: QgsVectorLayer) -> QgsVectorLayer: """ Assign scores to a grid layer and rasterize it. - Args: grid_layer (QgsVectorLayer): The grid layer representing the study area. buffered_layer (QgsVectorLayer): Buffered layer to evaluate intersections. index (int): Index for output file naming. - Returns: str: Path to the output raster file. """ @@ -234,45 +211,37 @@ def _score_grid(self, grid_layer: QgsVectorLayer, buffered_layer: QgsVectorLayer tag="GeoE3", level=Qgis.Info, ) - # Add a new attribute to the grid layer for storing the score grid_layer.startEditing() if not grid_layer.fields().indexFromName("score") >= 0: grid_layer.dataProvider().addAttributes([QgsField("score", QVariant.Int)]) grid_layer.updateFields() - # Assign scores based on intersection with the buffered layer for grid_feature in grid_layer.getFeatures(): grid_geom = grid_feature.geometry() max_score = 0 - for buffered_feature in buffered_layer.getFeatures(): buffered_geom = buffered_feature.geometry() intersection = grid_geom.intersection(buffered_geom) if intersection.isEmpty(): continue - if self.scoring_method == "binary": max_score = max(max_score, self.scores.get("intersects_buffer", 5)) continue - overlap_percent = (intersection.area() / grid_geom.area()) * 100 log_message( f"Overlap percentage: {overlap_percent}", tag="GeoE3", level=Qgis.Info, ) - # Determine score based on overlap percentage thresholds if self.scoring_method == "percentage_intersection": for min_pct, score in sorted(self.percentage_scores.items(), reverse=True): if overlap_percent >= min_pct: max_score = max(max_score, score) break - # Update the "score" attribute for the feature grid_feature.setAttribute("score", max_score) grid_layer.updateFeature(grid_feature) - grid_layer.commitChanges() return grid_layer diff --git a/geest/core/workflows/workflow_base.py b/geest/core/workflows/workflow_base.py index feb34872..de13ed2b 100644 --- a/geest/core/workflows/workflow_base.py +++ b/geest/core/workflows/workflow_base.py @@ -6,6 +6,7 @@ import datetime import os +import sqlite3 import traceback from abc import abstractmethod from typing import Optional @@ -37,6 +38,10 @@ subset_vector_layer, ) from geest.core.constants import GDAL_OUTPUT_DATA_TYPE +from geest.core.grid_column_utils import ( + rasterize_grid_column, + write_raster_values_to_grid, +) from geest.utilities import log_layer_count, log_message, resources_path @@ -116,7 +121,7 @@ def __init__( self.grid_layer = QgsVectorLayer(f"{self.gpkg_path}|layername=study_area_grid", "study_area_grid", "ogr") self.features_layer = None # set in concrete class if needed self.raster_layer = None # set in concrete class if needed - self.target_crs = self.bboxes_layer.crs() + self.target_crs = self._resolve_target_crs() self.result_file_key = "result_file" self.result_key = "result" @@ -128,9 +133,19 @@ def __init__( attrs["error_file"] = None attrs["execution_start_time"] = None attrs["execution_end_time"] = None - self.layer_id = self.attributes.get("id", "").lower().replace(" ", "_") + # Add prefix based on item role to avoid namespace collisions + raw_id = self.attributes.get("id", "").lower().replace(" ", "_") + role = self.item.role if hasattr(self.item, "role") else "" + if role == "dimension": + self.layer_id = f"dim_{raw_id}" + elif role == "factor": + self.layer_id = f"fac_{raw_id}" + else: + self.layer_id = raw_id # indicators keep raw ID self.aggregation = False self.analysis_mode = self.item.attribute("analysis_mode", "") + # Grid-first mode: if True, skip raster-to-grid sampling since grid is already populated + self.use_grid_first = False self.updateProgress(0.0) self.output_filename = self.attributes.get("output_filename", "") self.feedback.progressChanged.connect(self.updateProgress) @@ -163,6 +178,79 @@ def _study_area_bbox_4326(self) -> QgsRectangle: bbox = QgsCoordinateTransform.transformBoundingBox(transform, bbox) return bbox + def _resolve_target_crs(self) -> QgsCoordinateReferenceSystem: + """Resolve the target CRS from the study area GeoPackage. + + First tries ``self.bboxes_layer.crs()``. If the QGIS OGR provider + returns an invalid/empty CRS (which can happen when WAL journal files + leave stale shared-memory state), falls back to reading the CRS + directly from the ``gpkg_geometry_columns`` / ``gpkg_spatial_ref_sys`` + metadata tables via OGR SQL. + """ + crs = self.bboxes_layer.crs() + if crs.isValid() and crs.authid(): + return crs + + log_message( + "bboxes_layer CRS is invalid or empty — reading CRS from gpkg metadata", + tag="GeoE3", + level=Qgis.Warning, + ) + try: + from osgeo import ogr + + ds = ogr.Open(self.gpkg_path, 0) + if ds: + result = ds.ExecuteSQL( + "SELECT gc.srs_id, srs.organization, srs.organization_coordsys_id " + "FROM gpkg_geometry_columns gc " + "JOIN gpkg_spatial_ref_sys srs ON gc.srs_id = srs.srs_id " + "WHERE gc.table_name = 'study_area_bboxes' LIMIT 1" + ) + if result: + feat = result.GetNextFeature() + if feat: + org = feat.GetField("organization") + org_id = feat.GetField("organization_coordsys_id") + if org and org_id: + crs = QgsCoordinateReferenceSystem(f"{org}:{org_id}") + log_message( + f"Recovered CRS from gpkg metadata: {crs.authid()}", + tag="GeoE3", + level=Qgis.Info, + ) + ds.ReleaseResultSet(result) + ds = None + except Exception as e: + log_message( + f"Failed to read CRS from gpkg metadata: {e}", + tag="GeoE3", + level=Qgis.Critical, + ) + + if not crs.isValid(): + integrity_status = self._quick_check_gpkg() + raise ValueError( + f"Could not determine CRS for study area from {self.gpkg_path}. " + f"GeoPackage integrity check: {integrity_status}." + ) + return crs + + def _quick_check_gpkg(self) -> str: + """Run SQLite quick_check on the study area GeoPackage.""" + try: + connection = sqlite3.connect(self.gpkg_path) + try: + cursor = connection.cursor() + cursor.execute("PRAGMA quick_check;") + row = cursor.fetchone() + result = row[0] if row else "unknown" + return str(result) + finally: + connection.close() + except Exception as error: + return f"failed ({error})" + def _check_ghsl_layer_exists(self) -> bool: """Check if the GHSL settlements layer exists in the study area GeoPackage. @@ -345,6 +433,7 @@ def _process_features_for_area( current_bbox: QgsGeometry, area_features: QgsVectorLayer, index: int, + area_name: Optional[str] = None, ) -> str: """ Executes the actual workflow logic for a single area @@ -356,6 +445,7 @@ def _process_features_for_area( current_bbox: Bounding box of the above area. area_features: A vector layer of features to analyse that includes only features in the study area. index: Iteration / number of area being processed. + area_name: Name of the area being processed (for grid-first mode). Returns: A raster layer file path if processing completes successfully, False if canceled or failed. @@ -370,6 +460,7 @@ def _process_raster_for_area( current_bbox: QgsGeometry, area_raster: str, index: int, + area_name: Optional[str] = None, ): """ Executes the actual workflow logic for a single area using a raster. @@ -380,6 +471,7 @@ def _process_raster_for_area( current_bbox: Bounding box of the above area. area_raster: A raster layer of features to analyse that includes only bbox pixels in the study area. index: Index of the current area. + area_name: Name of the area being processed (for grid-first mode). Returns: Path to the reclassified raster. @@ -393,6 +485,7 @@ def _process_aggregate_for_area( clip_area: QgsGeometry, current_bbox: QgsGeometry, index: int, + area_name: Optional[str] = None, ): """ Executes the actual workflow logic for a single area using an aggregate. @@ -402,6 +495,7 @@ def _process_aggregate_for_area( clip_area: Polygon to clip the raster to which is aligned to cell edges. current_bbox: Bounding box of the above area. index: Index of the current area. + area_name: Name of the area being processed (for grid-first mode). Returns: Path to the reclassified raster. @@ -482,7 +576,7 @@ def execute(self) -> bool: try: total_areas = area_iterator.area_count() - for index, (current_area, clip_area, current_bbox, progress) in enumerate(area_iterator): + for index, (current_area, clip_area, current_bbox, progress, area_name) in enumerate(area_iterator): areas_processed += 1 message = f"{self.workflow_name} Processing area {index} with progress {progress:.2f}%" # noqa E231 self.updateStatus(f"Processing area {index + 1}/{total_areas}") @@ -522,6 +616,7 @@ def execute(self) -> bool: current_bbox=current_bbox, area_features=area_features, index=index, + area_name=area_name, ) elif not self.aggregation: # assumes we are processing a raster input area_raster = self._subset_raster_layer(bbox=current_bbox, index=index) @@ -531,6 +626,7 @@ def execute(self) -> bool: current_bbox=current_bbox, area_raster=area_raster, index=index, + area_name=area_name, ) elif self.aggregation: # we are processing an aggregate raster_output = self._process_aggregate_for_area( @@ -538,6 +634,12 @@ def execute(self) -> bool: clip_area=clip_area, current_bbox=current_bbox, index=index, + area_name=area_name, + ) + + if not raster_output: + raise RuntimeError( + f"{self.workflow_name} produced no raster output for area {area_name} (index {index})." ) # clip the area by its matching mask layer in study_area geopackage @@ -548,6 +650,30 @@ def execute(self) -> bool: index=index, ) output_rasters.append(masked_layer) + + # Write raster values to grid for this area + # Skip this step for grid-first workflows since grid was already populated + if not self.use_grid_first and masked_layer and os.path.exists(masked_layer): + self.updateStatus(f"Writing grid values for area {index + 1}...") + updated_cells = write_raster_values_to_grid( + gpkg_path=self.gpkg_path, + raster_path=masked_layer, + column_name=self.layer_id, + area_name=area_name, + ) + if updated_cells >= 0: + log_message( + f"Updated {updated_cells} grid cells for {self.layer_id} in area {area_name}", + tag="GeoE3", + level=Qgis.Info, + ) + else: + log_message( + f"Failed to update grid cells for {self.layer_id} in area {area_name}", + tag="GeoE3", + level=Qgis.Warning, + ) + # Note: We don't emit area iterator progress here because it would # override the sub-task progress in the Task Progress bar. # The sub-task progress (0-100%) is more useful to the user. @@ -893,38 +1019,111 @@ def _combine_rasters_to_vrt(self, rasters: list) -> None: role = self.item.role source_qml = resources_path("resources", "qml", f"{role}.qml") vrt_filepath = combine_rasters_to_vrt(rasters, self.target_crs, vrt_filepath, source_qml) - # if debug mode is off, remove all files except the VRT and the rasters it refers to + # if debug mode is off, remove all intermediate files if not int(setting(key="developer_mode", default=0)): - log_message("Debug mode is off. Removing all files except the VRT and the rasters it refers to.") - # Compile a list of all of the files in the workflow directory - recursively + log_message("Debug mode is off. Removing intermediate files, keeping only VRT-referenced rasters.") + # Build set of TIF filenames referenced by VRTs in this directory + # VRTs are locally generated XML — extract SourceFilename values via regex + import re + referenced_tifs = set() all_files = os.listdir(self.workflow_directory) - # Remove all files except the VRT, qml and the rasters it refers to - # loop through all files in the workflow directory + source_pattern = re.compile(r"]*>([^<]+)") for file in all_files: - file_path = os.path.join(self.workflow_directory, file) - if ( - not file.endswith(".vrt") # noqa W503 - and not file.endswith(".qml") # noqa W503 - and not file.endswith(".tif") # noqa W503 - and not file.endswith("error.txt") # noqa W503 - ): - log_message(f"Removing {file_path}") + if file.endswith(".vrt"): try: - os.remove(file_path) - except Exception as e: - log_message( - f"Failed to remove {file_path}: {e}", - tag="GeoE3", - level=Qgis.Warning, - ) - log_message( - traceback.format_exc(), - tag="GeoE3", - level=Qgis.Warning, - ) - continue + vrt_path = os.path.join(self.workflow_directory, file) + with open(vrt_path, "r") as f: + for match in source_pattern.finditer(f.read()): + referenced_tifs.add(os.path.basename(match.group(1))) + except Exception: # nosec B110 + pass # If VRT can't be read, keep all TIFs as fallback + + for file in all_files: + file_path = os.path.join(self.workflow_directory, file) + # Keep: VRTs, QMLs, error logs, and TIFs referenced by VRTs + if file.endswith(".vrt") or file.endswith(".qml") or file.endswith("error.txt"): + continue + if file.endswith(".tif") and file in referenced_tifs: + continue + # Skip subdirectories — they contain child workflow outputs + if os.path.isdir(file_path): + continue + # Delete intermediate files (unreferenced TIFs, shapefiles, etc.) + log_message(f"Removing {file_path}") + try: + os.remove(file_path) + except Exception as e: + log_message( + f"Failed to remove {file_path}: {e}", + tag="GeoE3", + level=Qgis.Warning, + ) else: log_message("Debug mode is on. Keeping all files in the workflow directory.") return vrt_filepath + + def _rasterize_grid_column( + self, + column_name: str, + bbox: QgsGeometry, + area_name: str, + index: int, + nodata: float = -9999.0, + ) -> Optional[str]: + """Rasterize a grid column to create a raster output. + + This method creates a raster from the study_area_grid column using + gdal_rasterize. It is used for grid-first workflows where results + are written to grid columns first, then rasterized for VRT generation. + + Args: + column_name: Name of the grid column to rasterize. + bbox: Bounding box geometry for the output raster extent. + area_name: Name of the area being processed. + index: Index of the area being processed (for output filename). + nodata: NoData value for the output raster. + + Returns: + Path to the output raster, or None on error. + """ + output_path = os.path.join( + self.workflow_directory, + f"{column_name}_from_grid_{index}.tif", + ) + + # Get extent from bbox + rect = bbox.boundingBox() + extent = (rect.xMinimum(), rect.yMinimum(), rect.xMaximum(), rect.yMaximum()) + + log_message( + f"Rasterizing grid column {column_name} for area {area_name}", + tag="GeoE3", + level=Qgis.Info, + ) + + success = rasterize_grid_column( + gpkg_path=self.gpkg_path, + column_name=column_name, + output_raster_path=output_path, + cell_size=self.cell_size_m, + extent=extent, + nodata=nodata, + area_name=area_name, + ) + + if success: + log_message( + f"Rasterized grid column {column_name} to {output_path}", + tag="GeoE3", + level=Qgis.Info, + ) + return output_path + else: + log_message( + f"Failed to rasterize grid column {column_name}", + tag="GeoE3", + level=Qgis.Warning, + ) + return None diff --git a/geest/gui/datasource_widget_factory.py b/geest/gui/datasource_widget_factory.py index b86db7ed..e314524e 100644 --- a/geest/gui/datasource_widget_factory.py +++ b/geest/gui/datasource_widget_factory.py @@ -16,6 +16,10 @@ EPLEXDataSourceWidget, FixedValueDataSourceWidget, RasterDataSourceWidget, + S2SEnvironmentalHazardsRasterDataSourceWidget, + S2SDataSourceWidget, + S2SEducationDataSourceWidget, + S2SNTLRasterDataSourceWidget, VectorAndFieldDataSourceWidget, VectorDataSourceWidget, ) @@ -71,6 +75,11 @@ def create_widget(widget_key: str, value: int, attributes: dict) -> Optional[Bas if widget_key == "use_single_buffer_point" and value == 1: return VectorDataSourceWidget(widget_key=cleaned_key, attributes=attributes) if widget_key == "use_polygon_per_cell" and value == 1: + analysis_scale = attributes.get("analysis_scale") + if analysis_scale == "regional": + if str(attributes.get("id", "")).strip().lower() == "education": + return S2SEducationDataSourceWidget(widget_key=cleaned_key, attributes=attributes) + return S2SDataSourceWidget(widget_key=cleaned_key, attributes=attributes) return VectorDataSourceWidget(widget_key=cleaned_key, attributes=attributes) if widget_key == "use_polyline_per_cell" and value == 1: return VectorDataSourceWidget(widget_key=cleaned_key, attributes=attributes) @@ -87,8 +96,14 @@ def create_widget(widget_key: str, value: int, attributes: dict) -> Optional[Bas if widget_key == "use_classify_safety_polygon_into_classes" and value == 1: return VectorAndFieldDataSourceWidget(widget_key=cleaned_key, attributes=attributes) if widget_key == "use_nighttime_lights" and value == 1: + analysis_scale = attributes.get("analysis_scale") + if analysis_scale == "regional": + return S2SNTLRasterDataSourceWidget(widget_key=cleaned_key, attributes=attributes) return RasterDataSourceWidget(widget_key=cleaned_key, attributes=attributes) if widget_key == "use_environmental_hazards" and value == 1: + analysis_scale = attributes.get("analysis_scale") + if analysis_scale == "regional": + return S2SEnvironmentalHazardsRasterDataSourceWidget(widget_key=cleaned_key, attributes=attributes) return RasterDataSourceWidget(widget_key=cleaned_key, attributes=attributes) if widget_key == "use_street_lights" and value == 1: return VectorDataSourceWidget(widget_key=cleaned_key, attributes=attributes) diff --git a/geest/gui/dialogs/factor_aggregation_dialog.py b/geest/gui/dialogs/factor_aggregation_dialog.py index 3fe2af32..be17bc63 100644 --- a/geest/gui/dialogs/factor_aggregation_dialog.py +++ b/geest/gui/dialogs/factor_aggregation_dialog.py @@ -4,6 +4,8 @@ This module contains functionality for factor aggregation dialog. """ +from typing import Optional, Sequence + from qgis.core import Qgis from qgis.PyQt.QtCore import QByteArray, QSettings, Qt, QUrl from qgis.PyQt.QtGui import QDesktopServices, QFont, QPixmap @@ -51,7 +53,14 @@ class FactorAggregationDialog(CustomBaseDialog): factor_data: Factor data. """ - def __init__(self, factor_name, factor_data, factor_item, parent=None): + def __init__( + self, + factor_name, + factor_data, + factor_item, + parent=None, + selected_guids: Optional[Sequence[str]] = None, + ): """🏗️ Initialize the instance. Args: @@ -69,6 +78,11 @@ def __init__(self, factor_name, factor_data, factor_item, parent=None): # Initialize dictionaries self.guids = self.tree_item.getFactorIndicatorGuids() + if selected_guids is not None: + selected_guid_set = set(selected_guids) + self.guids = [guid for guid in self.guids if guid in selected_guid_set] + if not self.guids: + self.guids = self.tree_item.getFactorIndicatorGuids() # If the indicators do not have a usable analysis mode set, iterate through them # and set it to the first available usable mode for guid in self.guids: @@ -151,7 +165,7 @@ def __init__(self, factor_name, factor_data, factor_item, parent=None): self.table.setColumnCount(6) self.table.setHorizontalHeaderLabels(["Input", "Indicator", "Weight 0-1", "Use", "GUID", ""]) self.table.horizontalHeader().setSectionResizeMode(0, QHeaderView.Stretch) - self.table.horizontalHeader().setSectionResizeMode(1, QHeaderView.Stretch) + self.table.horizontalHeader().setSectionResizeMode(1, QHeaderView.Fixed) self.table.horizontalHeader().setSectionResizeMode(2, QHeaderView.Fixed) self.table.horizontalHeader().setSectionResizeMode(3, QHeaderView.Fixed) self.table.horizontalHeader().setSectionResizeMode(4, QHeaderView.Stretch) @@ -170,6 +184,7 @@ def __init__(self, factor_name, factor_data, factor_item, parent=None): self.table.setColumnWidth(4, 50) self.table.setColumnWidth(6, 75) else: + self.table.setColumnWidth(1, 200) self.table.setColumnWidth(2, 100) self.table.setColumnWidth(3, 50) self.table.setColumnWidth(5, 75) @@ -359,6 +374,9 @@ def populate_table(self): row = guid_index item = self.tree_item.getItemByGuid(guid) attributes = item.attributes() + analysis_item = self.tree_item.parentItem.parentItem if self.tree_item.parentItem else None + if analysis_item: + attributes["analysis_scale"] = analysis_item.attribute("analysis_scale", "") log_message(f"Populating table for GUID: {guid}") log_message(f"Attributes: {item.attributesAsMarkdown()}") diff --git a/geest/gui/geoe3_dock.py b/geest/gui/geoe3_dock.py index e88332ec..47fef13e 100644 --- a/geest/gui/geoe3_dock.py +++ b/geest/gui/geoe3_dock.py @@ -5,6 +5,7 @@ """ import os +import json from typing import Optional from qgis.core import Qgis, QgsProject @@ -22,6 +23,7 @@ OpenProjectPanel, OrsPanel, RoadNetworkPanel, + S2SPanel, SetupPanel, TreePanel, ) @@ -37,10 +39,11 @@ SETUP_PANEL = 2 OPEN_PROJECT_PANEL = 3 CREATE_PROJECT_PANEL = 4 -ORS_PANEL = 5 -ROAD_NETWORK_PANEL = 6 -TREE_PANEL = 7 -HELP_PANEL = 8 +S2S_PANEL = 5 +ORS_PANEL = 6 +ROAD_NETWORK_PANEL = 7 +TREE_PANEL = 8 +HELP_PANEL = 9 class GeoE3Dock(QDockWidget): @@ -65,6 +68,7 @@ def __init__(self, parent: Optional[QWidget] = None, json_file: Optional[str] = self.initialised = False self._suppress_qgis_project_changed = False # Flag to prevent signal loop super().__init__(parent) + self.background_image = theme_background_image() # Get the plugin version from metadata.txt self.plugin_version = version() @@ -95,6 +99,7 @@ def __init__(self, parent: Optional[QWidget] = None, json_file: Optional[str] = self.road_network_widget: RoadNetworkPanel = RoadNetworkPanel() self.road_network_widget.set_message_bar(self.message_bar) # Pass message bar reference self.create_project_widget: CreateProjectPanel = CreateProjectPanel() + self.s2s_widget: S2SPanel = S2SPanel() self.ors_widget: OrsPanel = OrsPanel() self.tree_widget: TreePanel = TreePanel(json_file=self.json_file) help_widget: HelpPanel = HelpPanel() @@ -193,19 +198,30 @@ def __init__(self, parent: Optional[QWidget] = None, json_file: Optional[str] = lambda: self.stacked_widget.setCurrentIndex(SETUP_PANEL) ) - self.create_project_widget.switch_to_next_tab.connect( - # Switch to the next tab when the button is clicked - lambda: [ - self.stacked_widget.setCurrentIndex(ORS_PANEL), - ][ - -1 - ] # The [-1] ensures the lambda returns the last value - ) + self.create_project_widget.switch_to_next_tab.connect(self._open_next_panel_after_project_creation) self.create_project_widget.working_directory_changed.connect( - lambda: self.tree_widget.set_working_directory(self.create_project_widget.working_dir) + lambda _path: self.tree_widget.set_working_directory(self.create_project_widget.working_dir) + ) + self.create_project_widget.working_directory_changed.connect( + lambda _path: self.s2s_widget.set_working_directory(self.create_project_widget.working_dir) + ) + + # S2S_PANEL = 5 + # Create and add the "S2S" panel + + s2s_panel: QWidget = QWidget() + s2s_layout: QVBoxLayout = QVBoxLayout(s2s_panel) + s2s_layout.setContentsMargins(10, 10, 10, 10) + s2s_layout.addWidget(self.s2s_widget) + self.stacked_widget.addWidget(s2s_panel) + + self.s2s_widget.switch_to_previous_tab.connect( + lambda: self.stacked_widget.setCurrentIndex(CREATE_PROJECT_PANEL) ) - # ORS_PANEL = 5 + self.s2s_widget.switch_to_next_tab.connect(lambda: self.stacked_widget.setCurrentIndex(ORS_PANEL)) + + # ORS_PANEL = 6 # Create and add the "ORS" panel ors_panel: QWidget = QWidget() @@ -214,9 +230,7 @@ def __init__(self, parent: Optional[QWidget] = None, json_file: Optional[str] = ors_layout.addWidget(self.ors_widget) self.stacked_widget.addWidget(ors_panel) - self.ors_widget.switch_to_previous_tab.connect( - lambda: self.stacked_widget.setCurrentIndex(CREATE_PROJECT_PANEL) - ) + self.ors_widget.switch_to_previous_tab.connect(self._open_previous_panel_before_ors) self.ors_widget.switch_to_next_tab.connect(self._open_road_network_from_ors) @@ -319,7 +333,6 @@ def __init__(self, parent: Optional[QWidget] = None, json_file: Optional[str] = # Load the background image and style sheet # do this last so it applies to all the widgets - self.background_image = theme_background_image() main_widget.setStyleSheet(theme_stylesheet()) self.initialised = True @@ -329,9 +342,14 @@ def paintEvent(self, event): Args: event: Event. """ + background_image = getattr(self, "background_image", None) + if background_image is None or background_image.isNull(): + super().paintEvent(event) + return + with QPainter(self) as painter: # Calculate the scaling and cropping offsets - scaled_background = self.background_image.scaled(self.size(), Qt.KeepAspectRatioByExpanding) + scaled_background = background_image.scaled(self.size(), Qt.KeepAspectRatioByExpanding) # Calculate the offset to crop from top and right to keep bottom left anchored x_offset = max(0, scaled_background.width() - self.width()) @@ -426,6 +444,13 @@ def on_panel_changed(self, index: int) -> None: log_message("Switched to Create Project panel") elif index == ORS_PANEL: log_message("Switched to ORS panel") + elif index == S2S_PANEL: + if not self._is_regional_project_flow(): + self.stacked_widget.setCurrentIndex(ORS_PANEL) + return + working_directory = self.create_project_widget.working_dir or self.tree_widget.working_directory + self.s2s_widget.set_working_directory(working_directory) + log_message("Switched to S2S panel") elif index == ROAD_NETWORK_PANEL: working_directory = self.tree_widget.working_directory log_message(f"Setting road network panel working directory to: {working_directory}") @@ -460,3 +485,35 @@ def _open_road_network_from_ors(self) -> None: self.road_network_widget.set_crs( self.create_project_widget.crs(working_directory=self.create_project_widget.working_dir) ) + + def _open_next_panel_after_project_creation(self) -> None: + """Open the next panel after project creation based on analysis scale.""" + if self._is_regional_project_flow(): + self.stacked_widget.setCurrentIndex(S2S_PANEL) + else: + self.stacked_widget.setCurrentIndex(ORS_PANEL) + + def _open_previous_panel_before_ors(self) -> None: + """Open the previous panel before ORS based on analysis scale.""" + if self._is_regional_project_flow(): + self.stacked_widget.setCurrentIndex(S2S_PANEL) + else: + self.stacked_widget.setCurrentIndex(CREATE_PROJECT_PANEL) + + def _is_regional_project_flow(self) -> bool: + """Return True when current project analysis_scale is regional.""" + working_directory = self.create_project_widget.working_dir or self.tree_widget.working_directory + if not working_directory: + return False + + model_path = os.path.join(working_directory, "model.json") + if not os.path.exists(model_path): + return False + + try: + with open(model_path, "r", encoding="utf-8") as model_file: + model = json.load(model_file) + return model.get("analysis_scale") == "regional" + except Exception as error: + log_message(f"Failed reading model.json for panel routing: {error}", tag="GeoE3", level=Qgis.Warning) + return False diff --git a/geest/gui/overlays/layer_description.py b/geest/gui/overlays/layer_description.py index 5042cf68..3caea8e4 100644 --- a/geest/gui/overlays/layer_description.py +++ b/geest/gui/overlays/layer_description.py @@ -40,53 +40,59 @@ def paint(self, painter: QPainter, option=None, widget=None): option: Option. widget: Widget. """ - show_overlay = setting(key="show_overlay", default=False) - if not show_overlay: - return - # Get the label text from QSettings - label_text = QSettings().value("geoe3/overlay_label", "GeoE3 Overlay") - painter.setPen(QColor(0, 0, 0)) - font = QFont("Arial", 12, QFont.Bold) - painter.setFont(font) - painter.setRenderHint(QPainter.Antialiasing) - rect_x = 10 - rect_y = 10 - # Calculate width based on text - font_metrics = painter.fontMetrics() - text_width = font_metrics.horizontalAdvance(label_text) - text_height = font_metrics.height() - padding = 10 # Add some padding - - # Load geoe3 logo as SVG - icon = QIcon(resources_path("resources", "geoe3-main.svg")) - logo_x = 0 - logo_y = 0 - logo_width = 0 - if not icon.isNull(): - # Get pixmap from icon scaled to match text height - scaled_logo = icon.pixmap(text_height, text_height) - logo_width = scaled_logo.width() - logo_height = scaled_logo.height() - rect = QRectF( - rect_x, - rect_y, - padding + logo_width + padding + text_width + padding, - 50, - ) - - # Draw the logo on the left side of the rectangle - logo_x = int(rect_x + padding) - logo_y = int(rect_y + (rect.height() - logo_height) / 2) - else: - rect = QRectF(10, 10, text_width + padding, 50) - - painter.fillRect(rect, QColor(255, 255, 255, 128)) - painter.drawRect(rect) - - if not icon.isNull(): - painter.drawPixmap(logo_x, logo_y, scaled_logo) - # Modify the rectangle for text to start after the logo - new_left = logo_x + scaled_logo.width() - rect.setLeft(new_left) - # Set white background with 50% transparency - painter.drawText(rect, Qt.AlignmentFlag.AlignCenter, label_text) + # TEMPORARY: Disable the top-left layer description overlay. + # Re-enable the block below when issue triage is complete. + return + + # show_overlay = setting(key="show_overlay", default=False) + # if not show_overlay: + # return + # # Get the label text from QSettings + # label_text = QSettings().value("geoe3/overlay_label", "") + # if not label_text: + # return + # painter.setPen(QColor(0, 0, 0)) + # font = QFont("Arial", 12, QFont.Bold) + # painter.setFont(font) + # painter.setRenderHint(QPainter.Antialiasing) + # rect_x = 10 + # rect_y = 10 + # # Calculate width based on text + # font_metrics = painter.fontMetrics() + # text_width = font_metrics.horizontalAdvance(label_text) + # text_height = font_metrics.height() + # padding = 10 # Add some padding + # + # # Load geoe3 logo as SVG + # icon = QIcon(resources_path("resources", "geoe3-main.svg")) + # logo_x = 0 + # logo_y = 0 + # logo_width = 0 + # if not icon.isNull(): + # # Get pixmap from icon scaled to match text height + # scaled_logo = icon.pixmap(text_height, text_height) + # logo_width = scaled_logo.width() + # logo_height = scaled_logo.height() + # rect = QRectF( + # rect_x, + # rect_y, + # padding + logo_width + padding + text_width + padding, + # 50, + # ) + # + # # Draw the logo on the left side of the rectangle + # logo_x = int(rect_x + padding) + # logo_y = int(rect_y + (rect.height() - logo_height) / 2) + # else: + # rect = QRectF(10, 10, text_width + padding, 50) + # + # painter.fillRect(rect, QColor(255, 255, 255, 128)) + # painter.drawRect(rect) + # + # if not icon.isNull(): + # painter.drawPixmap(logo_x, logo_y, scaled_logo) + # # Modify the rectangle for text to start after the logo + # new_left = logo_x + scaled_logo.width() + # rect.setLeft(new_left) + # # Set white background with 50% transparency + # painter.drawText(rect, Qt.AlignmentFlag.AlignCenter, label_text) diff --git a/geest/gui/overlays/pie_chart.py b/geest/gui/overlays/pie_chart.py index d08cf37c..6f690822 100644 --- a/geest/gui/overlays/pie_chart.py +++ b/geest/gui/overlays/pie_chart.py @@ -66,6 +66,9 @@ def paint(self, painter: QPainter, option=None, widget=None): show_overlay = setting(key="show_pie_overlay", default=False) if not show_overlay: return + pie_data = QSettings().value("geoe3/pie_data", "") + if not pie_data: + return diameter = 100 image = QImage(diameter, diameter, QImage.Format_ARGB32) image.fill(Qt.GlobalColor.white) diff --git a/geest/gui/panels/__init__.py b/geest/gui/panels/__init__.py index c319ec9a..b208e5ac 100644 --- a/geest/gui/panels/__init__.py +++ b/geest/gui/panels/__init__.py @@ -14,5 +14,6 @@ from .open_project_panel import OpenProjectPanel from .ors_panel import OrsPanel from .road_network_panel import RoadNetworkPanel +from .s2s_panel import S2SPanel from .setup_panel import SetupPanel from .tree_panel import TreePanel diff --git a/geest/gui/panels/create_project_panel.py b/geest/gui/panels/create_project_panel.py index f2faf9ef..a34d32b3 100644 --- a/geest/gui/panels/create_project_panel.py +++ b/geest/gui/panels/create_project_panel.py @@ -6,24 +6,34 @@ import json import os +import sqlite3 import shutil +import time import traceback from qgis.core import ( Qgis, QgsCoordinateReferenceSystem, + QgsDistanceArea, QgsFeedback, QgsFieldProxyModel, QgsLayerTreeGroup, QgsMapLayerProxyModel, QgsProject, + QgsUnitTypes, QgsVectorLayer, ) from qgis.PyQt.QtCore import QSettings, pyqtSignal from qgis.PyQt.QtGui import QFont, QPixmap -from qgis.PyQt.QtWidgets import QFileDialog, QMessageBox, QWidget +from qgis.PyQt.QtWidgets import QComboBox, QFileDialog, QMessageBox, QWidget from geest.core import WorkflowQueueManager +from geest.core.h3_utils import ( + estimate_h3_cells_for_area, + h3_cell_area_km2, + suggest_coarser_resolution, + suggest_finer_resolution, +) from geest.core.tasks import StudyAreaProcessingTask, StudyAreaReportTask from geest.gui.widgets import CustomBannerLabel from geest.utilities import ( @@ -52,6 +62,9 @@ class CreateProjectPanel(FORM_CLASS, QWidget): # Signal to set the working directory working_directory_changed = pyqtSignal(str) + MIN_H3_ESTIMATED_CELLS = 3 + MAX_H3_ESTIMATED_CELLS = 200000 + def __init__(self): """🏗️ Initialize the instance.""" super().__init__() @@ -61,11 +74,37 @@ def __init__(self): self.working_dir = "" self.settings = QSettings() # Initialize QSettings to store and retrieve settings + self._last_map_refresh_ts = 0.0 # Dynamically load the .ui file self.setupUi(self) log_message("Loading setup panel") self.initUI() + def _should_skip_map_refresh(self) -> bool: + """Rate-limit map refresh attempts while study area is writing.""" + now = time.monotonic() + if now - self._last_map_refresh_ts < 0.5: + return True + self._last_map_refresh_ts = now + return False + + @staticmethod + def _gpkg_metadata_ready(gpkg_path: str) -> bool: + """Return True when required GeoPackage metadata tables exist.""" + try: + conn = sqlite3.connect(gpkg_path, timeout=0.2) + try: + cursor = conn.cursor() + cursor.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name IN ('gpkg_spatial_ref_sys','gpkg_contents')" + ) + names = {row[0] for row in cursor.fetchall()} + return "gpkg_spatial_ref_sys" in names and "gpkg_contents" in names + finally: + conn.close() + except Exception: + return False + def initUI(self): """⚙️ Initui.""" self.enable_widgets() # Re-enable widgets in case they were disabled @@ -82,14 +121,40 @@ def initUI(self): self.regional_scale.clicked.connect(lambda: self.spatial_scale_changed("regional")) self.national_scale.clicked.connect(lambda: self.spatial_scale_changed("national")) self.local_scale.clicked.connect(lambda: self.spatial_scale_changed("local")) + if self.regional_scale.isChecked(): + self.spatial_scale_changed("regional") + elif self.local_scale.isChecked(): + self.spatial_scale_changed("local") + else: + self.spatial_scale_changed("national") self.layer_combo.setFilters(QgsMapLayerProxyModel.PolygonLayer) # Regional scale uses H3 hexagonal grids (L6 resolution) # National and Local scales use square grids # Local mode enabled for National vs Local analysis implementation # self.local_scale.setEnabled(False) - self.regional_scale.setEnabled(False) + # self.regional_scale.setEnabled(False) self.regional_scale.setStyleSheet("QRadioButton:disabled { color: grey; }") + self.h3_resolution_label = self.label + self.h3_resolution_label.setText("H3 grid resolution") + self.h3_resolution_combo = QComboBox() + for resolution in range(16): + if resolution == 6: + self.h3_resolution_combo.addItem(f"{resolution} (recommended)", resolution) + else: + self.h3_resolution_combo.addItem(str(resolution), resolution) + self.h3_resolution_combo.setCurrentIndex(6) + self.h3_resolution_combo.setToolTip("Higher H3 resolutions are finer and may require significantly more time.") + + scale_layout = self.groupBox.layout() + scale_layout.addWidget(self.h3_resolution_combo, 3, 1, 1, 2) + if self.regional_scale.isChecked(): + self.spatial_scale_changed("regional") + elif self.local_scale.isChecked(): + self.spatial_scale_changed("local") + else: + self.spatial_scale_changed("national") + # Women Considerations toggle self.women_considerations_checkbox.stateChanged.connect(self.women_considerations_changed) # Initialize EPLEX widgets visibility based on checkbox state @@ -121,34 +186,46 @@ def initUI(self): # Ensure crs is set on first load self.layer_changed(self.layer_combo.currentLayer()) + def _refresh_boundary_crs_checkbox(self, layer: QgsVectorLayer) -> None: + """Refresh boundary CRS checkbox state for selected layer.""" + if not layer: + self.use_boundary_crs.setChecked(False) + self.use_boundary_crs.setEnabled(False) + return + + if layer.crs().authid() == "EPSG:4326": + self.use_boundary_crs.setChecked(False) + self.use_boundary_crs.setEnabled(False) + return + + self.use_boundary_crs.setEnabled(True) + def layer_changed(self, layer): """Slot to be called when the layer in the combo box changes. Args: layer: The new layer selected in the combo box. """ - log_message(f"Layer changed: {layer.name() if layer else 'None'}") - if self.crs() is None: - log_message( - "CRS is None, cannot set layer or field combo box.", - tag="GeoE3", - level=Qgis.Critical, - ) - self.crs_label.setText("Invalid CRS") - return - log_message(f"Layer crs: {layer.crs().authid() if layer else 'None'}") - if layer: - self.field_combo.setLayer(layer) - # Check if the layer has a valid CRS - if layer.crs().authid() == "EPSG:4326": - self.use_boundary_crs.setChecked(False) - self.use_boundary_crs.setEnabled(False) - else: - self.use_boundary_crs.setEnabled(True) + _ = layer + current_layer = self.layer_combo.currentLayer() + log_message(f"Layer changed: {current_layer.name() if current_layer else 'None'}") + + log_message(f"Layer crs: {current_layer.crs().authid() if current_layer else 'None'}") + if current_layer: + self.field_combo.setLayer(current_layer) else: self.field_combo.clear() - self.use_boundary_crs.setEnabled(False) - self.crs_label.setText(self.crs().authid()) + self._refresh_boundary_crs_checkbox(current_layer) + + try: + current_crs = self.crs() + if current_crs is not None and current_crs.authid(): + self.crs_label.setText(f"CRS: {current_crs.authid()}") + else: + self.crs_label.setText("CRS: Not set") + except Exception as error: + log_message(f"Could not resolve CRS after layer change: {error}", tag="GeoE3", level=Qgis.Warning) + self.crs_label.setText("CRS: Not set") def spatial_scale_changed(self, value: str): """Slot to be called when the spatial scale changes. @@ -160,18 +237,30 @@ def spatial_scale_changed(self, value: str): if value == "regional": # Regional scale uses H3 hexagonal grids (resolution L6) - fixed size self.cell_size_spinbox.hide() + if hasattr(self, "h3_resolution_label"): + self.h3_resolution_label.show() + if hasattr(self, "h3_resolution_combo"): + self.h3_resolution_combo.show() elif value == "national": self.cell_size_spinbox.show() self.description2.show() self.cell_size_spinbox.setValue(1000) self.cell_size_spinbox.setSingleStep(100) self.cell_size_spinbox.setSuffix(" m") + if hasattr(self, "h3_resolution_label"): + self.h3_resolution_label.hide() + if hasattr(self, "h3_resolution_combo"): + self.h3_resolution_combo.hide() elif value == "local": self.cell_size_spinbox.show() self.description2.show() self.cell_size_spinbox.setValue(100) self.cell_size_spinbox.setSingleStep(10) self.cell_size_spinbox.setSuffix(" m") + if hasattr(self, "h3_resolution_label"): + self.h3_resolution_label.hide() + if hasattr(self, "h3_resolution_combo"): + self.h3_resolution_combo.hide() def women_considerations_changed(self): """Slot to be called when the women considerations checkbox changes.""" @@ -205,6 +294,7 @@ def load_boundary(self): QgsProject.instance().addMapLayer(layer) self.layer_combo.setLayer(layer) self.field_combo.setLayer(layer) + self._refresh_boundary_crs_checkbox(self.layer_combo.currentLayer()) def create_new_project_folder(self): """⚙️ Create new project folder.""" @@ -250,6 +340,11 @@ def create_project(self): self.enable_widgets() return + if self.regional_scale.isChecked(): + if not self._validate_h3_preflight(layer, self.selected_h3_resolution()): + self.enable_widgets() + return + # Copy default model.json if not present default_model_path = resources_path("resources", "model.json") try: @@ -264,6 +359,7 @@ def create_project(self): model["analysis_cell_size_m"] = self.cell_size_spinbox.value() if self.regional_scale.isChecked(): model["analysis_scale"] = "regional" + model["analysis_h3_resolution"] = self.selected_h3_resolution() elif self.local_scale.isChecked(): model["analysis_scale"] = "local" else: @@ -290,6 +386,23 @@ def create_project(self): else: analysis_scale = "national" + h3_resolution = self.selected_h3_resolution() if analysis_scale == "regional" else None + + if analysis_scale == "regional" and h3_resolution is not None and h3_resolution >= 9: + reply = QMessageBox.question( + self, + "High H3 Resolution", + ( + f"H3 resolution {h3_resolution} can be very computationally expensive and may take " + "a long time to process.\n\nDo you want to continue?" + ), + QMessageBox.Yes | QMessageBox.No, + QMessageBox.No, + ) + if reply != QMessageBox.Yes: + self.enable_widgets() + return + processor = StudyAreaProcessingTask( layer=layer, field_name=field_name, @@ -298,6 +411,7 @@ def create_project(self): working_dir=self.working_dir, feedback=feedback, analysis_scale=analysis_scale, + h3_resolution=h3_resolution, ) # Hook up the QTask feedback signal to the progress bar # Measure overall task progress from the task object itself @@ -310,7 +424,7 @@ def create_project(self): feedback.progressChanged.connect(self.subtask_progress_updated) self.disable_widgets() if debug_env: - processor.process_study_area() + processor.run() else: self.queue_manager.add_task(processor) self.queue_manager.start_processing() @@ -495,7 +609,6 @@ def on_report_completed(self): self.child_progress_bar.setMaximum(100) self.child_progress_bar.setValue(100) self.child_progress_bar.setFormat("Complete") - self.enable_widgets() self.switch_to_next_tab.emit() @@ -509,7 +622,6 @@ def on_report_failed(self): self.child_progress_bar.setMaximum(100) self.child_progress_bar.setValue(0) self.child_progress_bar.setFormat("Report failed — continuing") - self.enable_widgets() self.switch_to_next_tab.emit() @@ -585,6 +697,8 @@ def set_font_size(self): self.create_project_directory_button.setFont(QFont("Arial", font_size)) self.load_boundary_button.setFont(QFont("Arial", font_size)) self.cell_size_spinbox.setFont(QFont("Arial", font_size)) + self.h3_resolution_label.setFont(QFont("Arial", font_size)) + self.h3_resolution_combo.setFont(QFont("Arial", font_size)) self.layer_combo.setFont(QFont("Arial", font_size)) self.field_combo.setFont(QFont("Arial", font_size)) @@ -599,6 +713,85 @@ def set_font_size(self): self.progress_bar.setFont(QFont("Arial", 9)) self.child_progress_bar.setFont(QFont("Arial", 9)) + def selected_h3_resolution(self) -> int: + """Return selected H3 resolution from regional dropdown.""" + selected_value = self.h3_resolution_combo.currentData() + if selected_value is None: + selected_value = self.h3_resolution_combo.currentText().split(" ")[0] + return int(selected_value) + + def _estimate_layer_area_km2(self, layer: QgsVectorLayer) -> float: + """Estimate total polygon area in square kilometers for selected input features.""" + if not layer: + return 0.0 + + distance_area = QgsDistanceArea() + distance_area.setSourceCrs(layer.crs(), QgsProject.instance().transformContext()) + if layer.crs().isGeographic(): + distance_area.setEllipsoid("WGS84") + else: + distance_area.setEllipsoid(layer.crs().ellipsoidAcronym() or "WGS84") + + area_m2 = 0.0 + if layer.selectedFeatureCount() > 0: + features = layer.getSelectedFeatures() + else: + features = layer.getFeatures() + + for feature in features: + geometry = feature.geometry() + if geometry is None or geometry.isEmpty(): + continue + area_m2 += abs(distance_area.measureArea(geometry)) + + return distance_area.convertAreaMeasurement(area_m2, QgsUnitTypes.AreaSquareKilometers) + + def _validate_h3_preflight(self, layer: QgsVectorLayer, h3_resolution: int) -> bool: + """Validate H3 configuration and block unsafe runs before processing.""" + area_km2 = self._estimate_layer_area_km2(layer) + if area_km2 <= 0: + QMessageBox.critical( + self, + "Invalid Study Area", + "Could not estimate study area size. Please verify polygon geometry and try again.", + ) + return False + + estimated_cells = estimate_h3_cells_for_area(area_km2, h3_resolution) + cell_area_km2 = h3_cell_area_km2(h3_resolution) + + if estimated_cells > self.MAX_H3_ESTIMATED_CELLS: + suggested = suggest_coarser_resolution(area_km2, self.MAX_H3_ESTIMATED_CELLS, h3_resolution) + QMessageBox.critical( + self, + "H3 Resolution Too Fine", + ( + f"Selected H3 resolution {h3_resolution} is too fine for this study area.\n\n" + f"Estimated area: {area_km2:,.2f} km²\n" + f"Approximate cell area: {cell_area_km2:,.6f} km²\n" + f"Estimated cells: {estimated_cells:,} (max allowed: {self.MAX_H3_ESTIMATED_CELLS:,})\n\n" + f"Please choose a coarser resolution, e.g. {suggested}." + ), + ) + return False + + if estimated_cells < self.MIN_H3_ESTIMATED_CELLS: + suggested = suggest_finer_resolution(area_km2, self.MIN_H3_ESTIMATED_CELLS, h3_resolution) + QMessageBox.critical( + self, + "H3 Resolution Too Coarse", + ( + f"Selected H3 resolution {h3_resolution} is too coarse for this study area.\n\n" + f"Estimated area: {area_km2:,.2f} km²\n" + f"Approximate cell area: {cell_area_km2:,.2f} km²\n" + f"Estimated cells: {estimated_cells} (minimum required: {self.MIN_H3_ESTIMATED_CELLS})\n\n" + f"Please choose a finer resolution, e.g. {suggested}." + ), + ) + return False + + return True + def add_bboxes_to_map(self): """Add the study area layers to the map. @@ -613,6 +806,37 @@ def add_bboxes_to_map(self): RuntimeError: If the GeoPackage cannot be opened for an unexpected reason. """ gpkg_path = os.path.join(self.working_dir, "study_area", "study_area.gpkg") + if self._should_skip_map_refresh(): + return + + if not os.path.exists(gpkg_path): + log_message( + f"GeoPackage not yet created: {gpkg_path}", + tag="GeoE3", + level=Qgis.Info, + ) + return + + try: + file_size = os.path.getsize(gpkg_path) + if file_size < 1024: # Less than 1KB suggests still initializing + log_message( + f"GeoPackage still initializing (size: {file_size} bytes)", + tag="GeoE3", + level=Qgis.Info, + ) + return + except OSError: + return + + if not self._gpkg_metadata_ready(gpkg_path): + log_message( + "GeoPackage metadata tables not ready yet; skipping map refresh.", + tag="GeoE3", + level=Qgis.Info, + ) + return + project = QgsProject.instance() # Check if 'GeoE3' group exists, otherwise create it @@ -626,29 +850,6 @@ def add_bboxes_to_map(self): "study_area_creation_status", ] for layer_name in layers: - # Check if GeoPackage file exists first - if not os.path.exists(gpkg_path): - log_message( - f"GeoPackage not yet created: {gpkg_path}", - tag="GeoE3", - level=Qgis.Info, - ) - return - - # Check if file size is stable (not being actively written) - # A very small file might still be initializing - try: - file_size = os.path.getsize(gpkg_path) - if file_size < 1024: # Less than 1KB suggests still initializing - log_message( - f"GeoPackage still initializing (size: {file_size} bytes)", - tag="GeoE3", - level=Qgis.Info, - ) - return - except OSError: - return # File might be locked - # Check if layer exists in GeoPackage from osgeo import ogr @@ -662,10 +863,19 @@ def add_bboxes_to_map(self): ds = None except RuntimeError as e: error_str = str(e).lower() - # Skip if database is busy or temporarily corrupted during writes - if "database is locked" in error_str or "malformed" in error_str: + # Skip if database is busy, temporarily corrupted, or still initializing + if ( + "database is locked" in error_str + or "malformed" in error_str + or "gpkg_spatial_ref_sys" in error_str + or "gpkg_contents" in error_str + or "not recognized as being in a supported file format" in error_str + or "unable to open database file" in error_str + or "readonly database" in error_str + or "required geopackage tables" in error_str + ): log_message( - f"Database busy or being written, skipping map refresh for {layer_name}", + f"Database busy or still initializing, skipping map refresh for {layer_name}", tag="GeoE3", level=Qgis.Info, ) diff --git a/geest/gui/panels/ors_panel.py b/geest/gui/panels/ors_panel.py index c237e62c..23110b5f 100644 --- a/geest/gui/panels/ors_panel.py +++ b/geest/gui/panels/ors_panel.py @@ -57,6 +57,7 @@ def initUI(self): self.status_label.setPixmap(QPixmap(resources_path("resources", "images", "ors-not-configured.png"))) self.next_button.clicked.connect(self.on_next_button_clicked) + self.previous_button.clicked.connect(self.on_previous_button_clicked) self.next_button.setEnabled(False) # Connect the rich text label's linkActivated signal to open URLs in browser self.description.linkActivated.connect(self.open_link_in_browser) diff --git a/geest/gui/panels/s2s_panel.py b/geest/gui/panels/s2s_panel.py new file mode 100644 index 00000000..54f67356 --- /dev/null +++ b/geest/gui/panels/s2s_panel.py @@ -0,0 +1,747 @@ +# -*- coding: utf-8 -*- +"""Space2Stats prefetch panel.""" + +import json +import os +from datetime import datetime +from typing import Dict, List + +from qgis.core import ( + QgsApplication, + Qgis, + QgsCoordinateReferenceSystem, + QgsCoordinateTransform, + QgsGeometry, + QgsProject, + QgsVectorLayer, +) +from qgis.PyQt.QtCore import QTimer, pyqtSignal +from qgis.PyQt.QtGui import QFont +from qgis.PyQt.QtWidgets import QMessageBox, QWidget + +from geest.core.constants import ( + DEFAULT_S2S_EDUCATION_URBANIZATION_FIELDS, + DEFAULT_S2S_ENV_HAZARD_FIELDS, + DEFAULT_S2S_NTL_FIELD, +) +from geest.core.s2s_task_gate import S2STaskGate +from geest.core.tasks import S2SDownloaderTask +from geest.gui.widgets import CustomBannerLabel +from geest.utilities import get_ui_class, linear_interpolation, log_message, resources_path + +FORM_CLASS = get_ui_class("s2s_panel_base.ui") + + +class S2SPanel(FORM_CLASS, QWidget): + """Panel that optionally prefetches S2S datasets after project creation.""" + + switch_to_next_tab = pyqtSignal() + switch_to_previous_tab = pyqtSignal() + PREFETCH_MAX_ATTEMPTS = 4 + PREFETCH_INTER_JOB_DELAY_MS = 750 + PREFETCH_CHUNK_SIZE = 3000 + PREFETCH_CHUNK_THRESHOLD = 50000 + + def __init__(self): + """Initialize panel and UI.""" + super().__init__() + self.setWindowTitle("GeoE3") + self.working_dir = "" + self._s2s_prefetch_jobs: List[Dict] = [] + self._s2s_prefetch_index = 0 + self._s2s_prefetch_warnings: List[str] = [] + self._s2s_prefetch_updates: List[Dict] = [] + self._s2s_prefetch_task = None + self._s2s_prefetch_error_for_current_task = False + self._s2s_prefetch_last_error_message = "" + self._s2s_prefetch_hex_ids: List[str] = [] + self._s2s_gate_token = None + self._s2s_prefetch_warning_keys = set() + self._s2s_prefetch_retry_timer_pending = False + + self.setupUi(self) + log_message("Loading S2S panel") + self.init_ui() + self.set_font_size() + + def init_ui(self) -> None: + """Initialize controls and signals.""" + self.custom_label = CustomBannerLabel( + "The Geospatial Enabling Environments for Employment Spatial Tool", + resources_path("resources", "geoe3-banner.png"), + ) + parent_layout = self.banner_label.parent().layout() + parent_layout.replaceWidget(self.banner_label, self.custom_label) + self.banner_label.deleteLater() + parent_layout.update() + + self.previous_button.clicked.connect(self.on_previous_button_clicked) + self.next_button.clicked.connect(self.on_next_button_clicked) + + self.progress_bar.setVisible(False) + self.processing_info_label.setVisible(False) + self.processing_info_label.setText("") + + def set_working_directory(self, working_dir: str) -> None: + """Set working directory and restore checkbox state from model.""" + self.working_dir = working_dir or "" + self._load_prefetch_state_from_model() + + def _load_prefetch_state_from_model(self) -> None: + """Load persisted prefetch checkbox value from model.json.""" + model = self._read_model() + self.prefetch_s2s_checkbox.setChecked(bool(model.get("s2s_prefetch_enabled", False))) + + def _read_model(self) -> dict: + """Read model.json for current project.""" + if not self.working_dir: + return {} + model_path = os.path.join(self.working_dir, "model.json") + if not os.path.exists(model_path): + return {} + try: + with open(model_path, "r", encoding="utf-8") as model_file: + return json.load(model_file) + except Exception as error: + log_message(f"Failed to read model.json in S2S panel: {error}", tag="GeoE3", level=Qgis.Warning) + return {} + + def _write_model(self, model: dict) -> bool: + """Write model.json for current project.""" + if not self.working_dir: + return False + model_path = os.path.join(self.working_dir, "model.json") + try: + with open(model_path, "w", encoding="utf-8") as model_file: + json.dump(model, model_file, indent=2) + return True + except Exception as error: + QMessageBox.warning(self, "S2S Fetch", f"Could not save model.json: {error}") + return False + + def on_previous_button_clicked(self) -> None: + """Return to project creation panel.""" + self.switch_to_previous_tab.emit() + + def on_next_button_clicked(self) -> None: + """Run optional S2S prefetch and continue to ORS panel.""" + model = self._read_model() + if not model: + self.switch_to_next_tab.emit() + return + + model["s2s_prefetch_enabled"] = self.prefetch_s2s_checkbox.isChecked() + if not self._write_model(model): + return + + if not self.prefetch_s2s_checkbox.isChecked(): + self.switch_to_next_tab.emit() + return + + if model.get("analysis_scale") != "regional": + self.switch_to_next_tab.emit() + return + + if not self._start_s2s_prefetch(model): + self.switch_to_next_tab.emit() + + def _start_s2s_prefetch(self, model: dict) -> bool: + """Start S2S prefetch for regional projects.""" + study_area_gpkg = os.path.join(self.working_dir, "study_area", "study_area.gpkg") + if not os.path.exists(study_area_gpkg): + log_message("S2S prefetch skipped: study area geopackage missing.", tag="GeoE3", level=Qgis.Warning) + return False + + aoi_layer = QgsVectorLayer(f"{study_area_gpkg}|layername=study_area_bboxes", "study_area_bboxes", "ogr") + if not aoi_layer.isValid() or aoi_layer.featureCount() == 0: + log_message("S2S prefetch skipped: study_area_bboxes unavailable.", tag="GeoE3", level=Qgis.Warning) + return False + + aoi_feature = self._build_aoi_feature(aoi_layer) + if not aoi_feature: + log_message("S2S prefetch skipped: failed to build AOI feature.", tag="GeoE3", level=Qgis.Warning) + return False + + self._s2s_prefetch_hex_ids = self._load_study_area_h3_indexes() + + jobs, warnings = self._prepare_s2s_prefetch_jobs(model) + self._s2s_prefetch_warnings = warnings + self._s2s_prefetch_updates = [] + self._s2s_prefetch_warning_keys = set() + self._s2s_prefetch_retry_timer_pending = False + + completed_jobs = set(self._load_prefetch_completed_jobs(model)) + pending_jobs: List[Dict] = [] + resumed_count = 0 + + for job in jobs: + job_with_mode = self._job_with_fetch_mode(job, model) + existing_output = self._existing_s2s_output_path(job_with_mode) + if self._is_existing_s2s_output_valid( + existing_output, + job_with_mode.get("fields", []), + job_with_mode.get("filename", ""), + ): + self._s2s_prefetch_updates.append( + { + "indicator_ids": job_with_mode["indicator_ids"], + "output_path": existing_output, + "metadata": job_with_mode["metadata"], + } + ) + completed_jobs.add(job_with_mode["filename"]) + self._clear_prefetch_job_state(job_with_mode["filename"], model=model) + resumed_count += 1 + continue + + if job_with_mode["filename"] in completed_jobs: + completed_jobs.remove(job_with_mode["filename"]) + + job_with_mode["attempt"] = 1 + pending_jobs.append(job_with_mode) + + self._store_prefetch_completed_jobs(model, completed_jobs) + self._write_model(model) + + if resumed_count: + self._append_prefetch_warning(f"Resuming prefetch: {resumed_count} datasets already available.") + + if not jobs: + if warnings: + QMessageBox.information(self, "S2S Fetch", "\n".join(warnings)) + return False + + if not pending_jobs: + self.processing_info_label.setText("All S2S datasets already available.") + self.processing_info_label.setVisible(True) + self.progress_bar.setVisible(True) + self.progress_bar.setMinimum(0) + self.progress_bar.setMaximum(100) + self.progress_bar.setValue(100) + self._finalize_s2s_prefetch() + return True + + self._s2s_prefetch_jobs = pending_jobs + self._s2s_prefetch_index = 0 + self._s2s_prefetch_task = None + self.processing_info_label.setText("Fetching S2S data for regional indicators...") + self.processing_info_label.setVisible(True) + self.progress_bar.setVisible(True) + self.progress_bar.setMinimum(0) + self.progress_bar.setMaximum(100) + self.progress_bar.setValue(0) + + self.previous_button.setEnabled(False) + self.next_button.setEnabled(False) + self.prefetch_s2s_checkbox.setEnabled(False) + + self._schedule_next_s2s_prefetch_job(aoi_feature, 0) + return True + + def _prepare_s2s_prefetch_jobs(self, model: dict) -> tuple[list, list]: + """Build a list of S2S prefetch jobs and non-blocking warnings.""" + jobs: List[Dict] = [] + warnings: List[str] = [] + + ntl_indicators: List[str] = [] + ntl_field = DEFAULT_S2S_NTL_FIELD + + for dimension in model.get("dimensions", []): + for factor in dimension.get("factors", []): + for indicator in factor.get("indicators", []): + indicator_id = str(indicator.get("id", "")).strip() + if not indicator_id: + continue + + if int(indicator.get("use_nighttime_lights", 0)) == 1: + ntl_indicators.append(indicator_id) + indicator_field = str(indicator.get("s2s_ntl_field") or "").strip() + if indicator_field: + ntl_field = indicator_field + + if int(indicator.get("use_environmental_hazards", 0)) == 1: + hazard_id = indicator_id.lower() + hazard_field = str(indicator.get("s2s_hazard_field") or "").strip() + if hazard_field == DEFAULT_S2S_NTL_FIELD: + hazard_field = "" + if not hazard_field: + hazard_field = DEFAULT_S2S_ENV_HAZARD_FIELDS.get(hazard_id, "") + if not hazard_field: + warnings.append(f"Skipped {indicator_id}: no S2S hazard field configured.") + continue + jobs.append( + { + "type": "hazard", + "indicator_ids": [indicator_id], + "fields": [hazard_field], + "filename": f"s2s_environmental_hazards_{hazard_id}", + "metadata": {"s2s_hazard_field": hazard_field, "s2s_ntl_field": ""}, + } + ) + + if int(indicator.get("use_polygon_per_cell", 0)) == 1: + fields = indicator.get("s2s_fields", []) + if isinstance(fields, str): + fields = [token.strip() for token in fields.split(",") if token.strip()] + elif isinstance(fields, list): + fields = [str(token).strip() for token in fields if str(token).strip()] + else: + fields = [] + + if not fields and indicator_id.lower() == "education": + fields = list(DEFAULT_S2S_EDUCATION_URBANIZATION_FIELDS) + + unique_fields = [] + for field in fields: + if field not in unique_fields: + unique_fields.append(field) + + if not unique_fields: + warnings.append(f"Skipped {indicator_id}: no s2s_fields configured.") + continue + + sanitized_id = indicator_id.lower().replace(" ", "_").replace("-", "_") + filename = f"s2s_polygon_per_cell_{sanitized_id}" + if indicator_id.lower() == "education": + filename = "s2s_education" + jobs.append( + { + "type": "polygon_per_cell", + "indicator_ids": [indicator_id], + "fields": unique_fields, + "filename": filename, + "metadata": { + "s2s_fields": unique_fields, + "s2s_fields_text": ",".join(unique_fields), + }, + } + ) + + if ntl_indicators: + jobs.insert( + 0, + { + "type": "nighttime_lights", + "indicator_ids": ntl_indicators, + "fields": [ntl_field], + "filename": "s2s_nighttime_lights", + "metadata": {"s2s_ntl_field": ntl_field}, + }, + ) + + return jobs, warnings + + def _run_next_s2s_prefetch_job(self, aoi_feature: dict) -> None: + """Run next queued S2S prefetch job.""" + if self._s2s_prefetch_index >= len(self._s2s_prefetch_jobs): + self._finalize_s2s_prefetch() + return + + job = self._s2s_prefetch_jobs[self._s2s_prefetch_index] + job_index = self._s2s_prefetch_index + 1 + total = len(self._s2s_prefetch_jobs) + mode_text = " (chunked)" if job.get("fetch_mode") == "hex_ids" else "" + self.processing_info_label.setText(f"Fetching S2S dataset {job_index}/{total}: {job['filename']}{mode_text}") + self.progress_bar.setFormat(f"S2S {job_index}/{total}: %p%") + + self._s2s_prefetch_task = S2SDownloaderTask( + aoi=aoi_feature, + fields=job["fields"], + working_dir=self.working_dir, + filename=job["filename"], + spatial_join_method="centroid", + geometry="point", + delete_existing=True, + mode=job.get("fetch_mode", "aoi"), + hex_ids=job.get("hex_ids"), + chunk_size=job.get("chunk_size", self.PREFETCH_CHUNK_SIZE), + start_chunk_index=job.get("start_chunk_index", 0), + append_existing=bool(job.get("start_chunk_index", 0) > 0), + ) + self._s2s_prefetch_task.progress_updated.connect(self._on_s2s_prefetch_progress_message) + self._s2s_prefetch_task.progressChanged.connect(self._on_s2s_prefetch_progress_value) + self._s2s_prefetch_task.error_occurred.connect(self._on_s2s_prefetch_error) + if job.get("fetch_mode") == "hex_ids": + self._s2s_prefetch_task.chunk_completed.connect( + lambda current_chunk, total_chunks, current_job=job: self._on_s2s_prefetch_chunk_completed( + current_job, + current_chunk, + total_chunks, + ) + ) + self._s2s_prefetch_error_for_current_task = False + self._s2s_prefetch_last_error_message = "" + self._s2s_prefetch_task.taskCompleted.connect( + lambda aoi=aoi_feature, current_job=job: self._on_s2s_prefetch_task_completed(current_job, aoi) + ) + self._s2s_prefetch_task.taskTerminated.connect( + lambda aoi=aoi_feature, current_job=job: self._on_s2s_prefetch_task_terminated(current_job, aoi) + ) + + gate_label = f"prefetch:{job.get('filename', '')}" + token = S2STaskGate.acquire(gate_label) + if not token: + active = S2STaskGate.active_label() or "another panel" + if active == gate_label: + self.processing_info_label.setText(f"S2S prefetch already running for {job.get('filename', '')}.") + else: + self.processing_info_label.setText(f"S2S prefetch waiting: another S2S download is running ({active}).") + self._schedule_next_s2s_prefetch_job(aoi_feature, 3000) + return + self._s2s_gate_token = token + + QgsApplication.taskManager().addTask(self._s2s_prefetch_task) + + def _on_s2s_prefetch_progress_message(self, message: str) -> None: + """Update status label with S2S task progress text.""" + self.processing_info_label.setText(message) + + def _on_s2s_prefetch_progress_value(self, progress: float) -> None: + """Update progress bar from S2S task progress.""" + self.progress_bar.setValue(int(progress)) + + def _on_s2s_prefetch_chunk_completed(self, job: dict, current_chunk: int, total_chunks: int) -> None: + """Persist chunk progress for resumable chunked prefetch jobs.""" + model = self._read_model() + if not model: + return + + state = { + "mode": "hex_ids", + "total_chunks": int(total_chunks), + "next_chunk_index": int(current_chunk), + "chunk_size": int(job.get("chunk_size", self.PREFETCH_CHUNK_SIZE)), + "updated_at": datetime.utcnow().isoformat() + "Z", + } + self._store_prefetch_job_state(model, job.get("filename", ""), state) + self._write_model(model) + + def _on_s2s_prefetch_error(self, message: str) -> None: + """Record non-blocking S2S prefetch errors.""" + self._s2s_prefetch_error_for_current_task = True + self._s2s_prefetch_last_error_message = message + + def _on_s2s_prefetch_task_completed(self, job: dict, aoi_feature: dict) -> None: + """Handle successful S2S prefetch task and run next.""" + self._release_s2s_gate() + output_path = os.path.join(self.working_dir, "study_area", f"{job['filename']}.gpkg") + if os.path.exists(output_path): + self._s2s_prefetch_updates.append( + { + "indicator_ids": job["indicator_ids"], + "output_path": output_path, + "metadata": job["metadata"], + } + ) + self._mark_prefetch_job_completed(job["filename"]) + self._clear_prefetch_job_state(job["filename"]) + else: + self._append_prefetch_warning(f"S2S output not found for {job['filename']}.") + + self._s2s_prefetch_index += 1 + self._schedule_next_s2s_prefetch_job(aoi_feature, self.PREFETCH_INTER_JOB_DELAY_MS) + + def _on_s2s_prefetch_task_terminated(self, job: dict, aoi_feature: dict) -> None: + """Handle terminated S2S prefetch tasks and continue queue.""" + self._release_s2s_gate() + if self._s2s_prefetch_error_for_current_task and self._is_transient_prefetch_error( + self._s2s_prefetch_last_error_message + ): + attempt = int(job.get("attempt", 1)) + if attempt < self.PREFETCH_MAX_ATTEMPTS: + next_attempt = attempt + 1 + delay_ms = self._retry_delay_ms(next_attempt) + job["attempt"] = next_attempt + self.processing_info_label.setText( + f"S2S unavailable for {job['filename']} - retrying " + f"{next_attempt}/{self.PREFETCH_MAX_ATTEMPTS} in {delay_ms / 1000:.1f}s..." + ) + self._schedule_next_s2s_prefetch_job(aoi_feature, delay_ms) + return + + if self._s2s_prefetch_error_for_current_task: + self._append_prefetch_warning( + self._s2s_prefetch_last_error_message or f"S2S prefetch failed: {job['filename']}" + ) + else: + self._append_prefetch_warning(f"S2S prefetch task terminated: {job['filename']}") + + self._s2s_prefetch_index += 1 + self._schedule_next_s2s_prefetch_job(aoi_feature, self.PREFETCH_INTER_JOB_DELAY_MS) + + def _schedule_next_s2s_prefetch_job(self, aoi_feature: dict, delay_ms: int) -> None: + """Schedule next prefetch job with optional delay.""" + delay = max(0, int(delay_ms)) + if delay == 0: + self._s2s_prefetch_retry_timer_pending = False + self._run_next_s2s_prefetch_job(aoi_feature) + return + + if self._s2s_prefetch_retry_timer_pending: + return + + self._s2s_prefetch_retry_timer_pending = True + + def _run_delayed(): + self._s2s_prefetch_retry_timer_pending = False + self._run_next_s2s_prefetch_job(aoi_feature) + + QTimer.singleShot(delay, _run_delayed) + + def _release_s2s_gate(self) -> None: + """Release global S2S gate lock for panel prefetch task.""" + if self._s2s_gate_token: + S2STaskGate.release(self._s2s_gate_token) + self._s2s_gate_token = None + + def _append_prefetch_warning(self, message: str) -> None: + """Append a warning once, preventing duplicate warning spam.""" + normalized = str(message or "").strip() + if not normalized: + return + if normalized in self._s2s_prefetch_warning_keys: + return + self._s2s_prefetch_warning_keys.add(normalized) + self._s2s_prefetch_warnings.append(normalized) + + def _existing_s2s_output_path(self, job: dict) -> str: + """Return expected output path for a prefetch job.""" + return os.path.join(self.working_dir, "study_area", f"{job['filename']}.gpkg") + + @staticmethod + def _is_existing_s2s_output_valid(output_path: str, fields: List[str], layer_name: str) -> bool: + """Return True when an existing S2S output has required fields and features.""" + if not output_path or not os.path.exists(output_path): + return False + + layer = QgsVectorLayer(f"{output_path}|layername={layer_name}", layer_name, "ogr") + if not layer.isValid(): + layer = QgsVectorLayer(output_path, layer_name, "ogr") + if not layer.isValid() or layer.featureCount() <= 0: + return False + + required_fields = ["hex_id"] + [field for field in fields if field != "hex_id"] + layer_fields = layer.fields() + for field in required_fields: + if layer_fields.indexFromName(field) == -1: + return False + return True + + @staticmethod + def _is_transient_prefetch_error(message: str) -> bool: + """Return True when a prefetch error should be retried.""" + lowered = str(message or "").lower() + return ( + "503" in lowered + or "temporarily unavailable" in lowered + or "server error (502)" in lowered + or "server error (504)" in lowered + or "no http status code" in lowered + or "timed out" in lowered + or "timeout" in lowered + or "connection" in lowered + ) + + @staticmethod + def _retry_delay_ms(attempt: int) -> int: + """Return exponential retry delay for a retry attempt number.""" + if attempt <= 2: + return 2000 + if attempt == 3: + return 5000 + return 10000 + + @staticmethod + def _load_prefetch_completed_jobs(model: dict) -> List[str]: + """Read persisted completed prefetch job filenames from model.""" + completed = model.get("s2s_prefetch_completed_jobs", []) + if not isinstance(completed, list): + return [] + return [str(name).strip() for name in completed if str(name).strip()] + + @staticmethod + def _store_prefetch_completed_jobs(model: dict, completed_jobs) -> None: + """Persist completed prefetch job filenames in model.""" + sorted_names = sorted({str(name).strip() for name in completed_jobs if str(name).strip()}) + model["s2s_prefetch_completed_jobs"] = sorted_names + + @staticmethod + def _load_prefetch_job_state(model: dict, filename: str) -> dict: + """Read persisted prefetch state for a specific job filename.""" + if not filename: + return {} + state = model.get("s2s_prefetch_job_state", {}) + if not isinstance(state, dict): + return {} + job_state = state.get(filename, {}) + return job_state if isinstance(job_state, dict) else {} + + @staticmethod + def _store_prefetch_job_state(model: dict, filename: str, state: dict) -> None: + """Persist prefetch checkpoint state for a specific job.""" + if not filename: + return + all_state = model.get("s2s_prefetch_job_state", {}) + if not isinstance(all_state, dict): + all_state = {} + all_state[filename] = state + model["s2s_prefetch_job_state"] = all_state + + def _clear_prefetch_job_state(self, filename: str, model: dict = None) -> None: + """Clear persisted checkpoint state for a specific job.""" + if not filename: + return + + model_in_use = model if model is not None else self._read_model() + if not model_in_use: + return + + all_state = model_in_use.get("s2s_prefetch_job_state", {}) + if not isinstance(all_state, dict) or filename not in all_state: + return + + all_state.pop(filename, None) + model_in_use["s2s_prefetch_job_state"] = all_state + if model is None: + self._write_model(model_in_use) + + def _load_study_area_h3_indexes(self) -> List[str]: + """Load H3 indexes from study_area_grid for chunked prefetch mode.""" + gpkg_path = os.path.join(self.working_dir, "study_area", "study_area.gpkg") + layer = QgsVectorLayer(f"{gpkg_path}|layername=study_area_grid", "study_area_grid", "ogr") + if not layer.isValid() or layer.featureCount() == 0: + return [] + + field_index = layer.fields().indexFromName("h3_index") + if field_index == -1: + return [] + + values = set() + for feature in layer.getFeatures(): + hex_id = str(feature["h3_index"] or "").strip() + if hex_id: + values.add(hex_id) + return sorted(values) + + def _job_with_fetch_mode(self, job: dict, model: dict) -> dict: + """Return a prefetch job enriched with fetch mode and resume state.""" + enriched = dict(job) + use_chunked = self._should_use_chunked_prefetch() + + if use_chunked: + state = self._load_prefetch_job_state(model, enriched.get("filename", "")) + start_chunk_index = int(state.get("next_chunk_index", 0)) if isinstance(state, dict) else 0 + enriched["fetch_mode"] = "hex_ids" + enriched["hex_ids"] = list(self._s2s_prefetch_hex_ids) + enriched["chunk_size"] = self.PREFETCH_CHUNK_SIZE + enriched["start_chunk_index"] = max(0, start_chunk_index) + else: + enriched["fetch_mode"] = "aoi" + enriched["start_chunk_index"] = 0 + + return enriched + + def _should_use_chunked_prefetch(self) -> bool: + """Return True when H3 coverage is large enough to use chunked S2S mode.""" + return len(self._s2s_prefetch_hex_ids) > self.PREFETCH_CHUNK_THRESHOLD + + def _mark_prefetch_job_completed(self, filename: str) -> None: + """Mark a prefetch job as completed and persist to model.""" + model = self._read_model() + if not model: + return + + completed_jobs = set(self._load_prefetch_completed_jobs(model)) + completed_jobs.add(filename) + self._store_prefetch_completed_jobs(model, completed_jobs) + self._write_model(model) + + def _finalize_s2s_prefetch(self) -> None: + """Write S2S prefetch metadata into model and continue flow.""" + model = self._read_model() + if model: + try: + self._apply_s2s_updates_to_model(model, self._s2s_prefetch_updates) + self._write_model(model) + except Exception as error: + self._append_prefetch_warning(f"Failed to store S2S prefetch metadata: {error}") + + self.progress_bar.setValue(100) + self.progress_bar.setFormat("S2S fetch complete") + self.processing_info_label.setText("S2S fetch completed.") + + if self._s2s_prefetch_warnings: + QMessageBox.information(self, "S2S Fetch", "\n".join(self._s2s_prefetch_warnings)) + + self.previous_button.setEnabled(True) + self.next_button.setEnabled(True) + self.prefetch_s2s_checkbox.setEnabled(True) + self.switch_to_next_tab.emit() + + @staticmethod + def _apply_s2s_updates_to_model(model: dict, updates: List[Dict]) -> None: + """Persist S2S output metadata into matching indicator attributes.""" + updates_by_indicator: Dict[str, List[Dict]] = {} + for update in updates: + for indicator_id in update.get("indicator_ids", []): + updates_by_indicator.setdefault(indicator_id, []).append(update) + + for dimension in model.get("dimensions", []): + for factor in dimension.get("factors", []): + for indicator in factor.get("indicators", []): + indicator_id = indicator.get("id") + if indicator_id not in updates_by_indicator: + continue + for update in updates_by_indicator[indicator_id]: + indicator["s2s_output_path"] = update["output_path"] + indicator["s2s_spatial_join_method"] = "centroid" + for key, value in update.get("metadata", {}).items(): + indicator[key] = value + + @staticmethod + def _build_aoi_feature(layer: QgsVectorLayer) -> dict: + """Build a WGS84 GeoJSON AOI feature from a vector layer.""" + geometries = [] + source_crs = layer.crs() + target_crs = QgsCoordinateReferenceSystem("EPSG:4326") + transform = None + if source_crs.isValid() and source_crs != target_crs: + transform = QgsCoordinateTransform(source_crs, target_crs, QgsProject.instance()) + + for feature in layer.getFeatures(): + geometry = feature.geometry() + if not geometry or geometry.isEmpty(): + continue + transformed_geometry = QgsGeometry(geometry) + if transform is not None: + transformed_geometry.transform(transform) + geometries.append(transformed_geometry) + + if not geometries: + return {} + + union_geometry = QgsGeometry.unaryUnion(geometries) + if not union_geometry or union_geometry.isEmpty(): + return {} + + return { + "type": "Feature", + "geometry": json.loads(union_geometry.asJson()), + "properties": {}, + } + + def resizeEvent(self, event): + """Handle resize events for adaptive font sizing.""" + self.set_font_size() + super().resizeEvent(event) + + def set_font_size(self): + """Set responsive font sizes for labels and controls.""" + font_size = int(linear_interpolation(self.description.rect().width(), 12, 16, 400, 600)) + font = QFont("Arial", font_size) + + self.description.setFont(font) + self.prefetch_s2s_checkbox.setFont(font) + self.prefetch_s2s_description.setFont(font) + self.processing_info_label.setFont(font) + self.progress_bar.setFont(QFont("Arial", 9)) diff --git a/geest/gui/panels/tree_panel.py b/geest/gui/panels/tree_panel.py index f95f4ddc..83fa5dca 100644 --- a/geest/gui/panels/tree_panel.py +++ b/geest/gui/panels/tree_panel.py @@ -23,6 +23,7 @@ QgsProject, QgsVectorLayer, ) +from qgis.gui import QgsMessageBar from qgis.PyQt.QtCore import QModelIndex, QPoint, QSettings, Qt, pyqtSignal, pyqtSlot from qgis.PyQt.QtGui import QMovie from qgis.PyQt.QtWidgets import ( @@ -45,7 +46,6 @@ QWidget, ) from qgis.utils import iface -from qgis.gui import QgsMessageBar from geest.core import JsonTreeItem, WorkflowQueueManager from geest.core.algorithms import ( @@ -56,10 +56,12 @@ SubnationalAggregationProcessingTask, WEEByPopulationScoreProcessingTask, ) +from geest.core.constants import MAX_FEATURES_FOR_VECTOR from geest.core.reports import StudyAreaReport -from geest.core.tasks import AnalysisReportTask from geest.core.settings import set_setting, setting -from geest.core.utilities import add_to_map, validate_network_layer +from geest.core.tasks import AnalysisReportTask +from geest.core.women_considerations import resolve_women_enabling_for_factor +from geest.core.utilities import add_grid_layer_to_map, add_to_map, validate_network_layer from geest.gui.dialogs import ( AnalysisAggregationDialog, DimensionAggregationDialog, @@ -136,19 +138,19 @@ def __init__(self, parent=None, json_file=None): """ QPushButton { background-color: qlineargradient(x1:0, y1:0, x2:0, y2:1, - stop:0 #b8dce3, stop:1 #8ec8d0); - color: #000; - border: 1px solid #6fa8b0; + stop:0 #3E799B, stop:1 #2d5a75); + color: #ffffff; + border: 1px solid #2d5a75; border-radius: 3px; padding: 4px 12px; } QPushButton:hover { background-color: qlineargradient(x1:0, y1:0, x2:0, y2:1, - stop:0 #c8e8ef, stop:1 #9ed8e0); + stop:0 #4a8bb0, stop:1 #3E799B); } QPushButton:pressed { background-color: qlineargradient(x1:0, y1:0, x2:0, y2:1, - stop:0 #8ec8d0, stop:1 #b8dce3); + stop:0 #2d5a75, stop:1 #3E799B); } """ ) @@ -327,7 +329,7 @@ def on_item_double_clicked(self, index): return if item.role == "indicator": - self.edit_factor_aggregation(item.parent()) + self.edit_factor_aggregation(item.parent(), selected_guids=[item.guid]) elif item.role == "factor": self.edit_factor_aggregation(item) elif item.role == "dimension": @@ -348,10 +350,25 @@ def on_item_clicked(self, index: QModelIndex): show_layer_on_click = setting(key="show_layer_on_click", default=True) if show_layer_on_click: - add_to_map(item) - show_overlay = setting(key="show_overlay", default=False) - if show_overlay: - QSettings().setValue("geoe3/overlay_label", item.data(0)) + if item.role == "dimension": + column_name = f"dim_{item.attribute('id').lower().replace(' ', '_').replace('-', '_')}" + elif item.role == "factor": + column_name = f"fac_{item.attribute('id').lower().replace(' ', '_').replace('-', '_')}" + elif item.role == "indicator": + column_name = item.attribute("id").lower().replace(" ", "_").replace("-", "_") + elif item.role == "analysis": + column_name = "geoe3" # Analysis aggregation uses geoe3 column + else: + column_name = None + + if column_name is None or self._get_render_strategy() == "raster": + add_to_map(item) + else: + add_grid_layer_to_map(item, column_name, self.working_directory) + # TEMPORARY: Disable writing layer name into top-left overlay label. + # show_overlay = setting(key="show_overlay", default=False) + # if show_overlay: + # QSettings().setValue("geoe3/overlay_label", item.data(0)) show_pie = setting(key="show_pie_overlay", default=False) if show_pie: # TODO - calculate the pie data @@ -359,10 +376,14 @@ def on_item_clicked(self, index: QModelIndex): def on_previous_button_clicked(self): """⚙️ On previous button clicked.""" + QSettings().setValue("geoe3/overlay_label", "") + QSettings().setValue("geoe3/pie_data", "") self.switch_to_previous_tab.emit() def on_next_button_clicked(self): """⚙️ On next button clicked.""" + QSettings().setValue("geoe3/overlay_label", "") + QSettings().setValue("geoe3/pie_data", "") self.switch_to_next_tab.emit() def clear_item(self): @@ -470,6 +491,8 @@ def working_directory_changed(self, new_directory): self.load_json() # sets the class member json_data self.model.loadJsonData(self.json_data) self.apply_women_considerations_logic() # Hide factors based on women considerations + # Hide factors that have only a single indicator (indicator shown in their place) + self.model.toggle_single_child_factors_visibility(hide_single_child=True) self.treeView.expandAll() log_message(f"Loaded model.json from {model_path}") @@ -521,6 +544,8 @@ def working_directory_changed(self, new_directory): self.load_json() self.model.loadJsonData(self.json_data) self.apply_women_considerations_logic() # Hide factors based on women considerations + # Hide factors that have only a single indicator (indicator shown in their place) + self.model.toggle_single_child_factors_visibility(hide_single_child=True) self.treeView.expandAll() # Collapse any factors that have only a single indicator self.treeView.collapse_single_nodes() @@ -534,6 +559,8 @@ def set_working_directory(self, working_directory): """ if working_directory: self.working_directory = working_directory + QSettings().setValue("geoe3/overlay_label", "") + QSettings().setValue("geoe3/pie_data", "") self.working_directory_changed(working_directory) @pyqtSlot() @@ -586,7 +613,7 @@ def set_ghsl_layer_path(self, ghsl_layer_path: str): """⚙️ Set ghsl layer path. Args: - ghsl_layer_path: Ghsl layer path. + ghsl_layer_path: GHSL layer path. """ if ghsl_layer_path: log_message(f"Setting ghsl_layer_path in model to {ghsl_layer_path}") @@ -607,6 +634,7 @@ def save_json_to_working_directory(self): tag="GeoE3", level=Qgis.Warning, ) + return try: json_data = self.model.to_json() @@ -702,8 +730,9 @@ def apply_women_considerations_logic(self): if not factor: continue - # Read women_enabling attribute from factor (data-driven approach) - women_enabling = factor.attribute("women_enabling", 0) + women_enabling = resolve_women_enabling_for_factor( + factor.attribute("id", ""), factor.attribute("women_enabling", 0) + ) factor_name = factor.data(0) # Determine enabled state based on women_enabling value @@ -764,6 +793,8 @@ def load_json_from_file(self): self.load_json() self.model.loadJsonData(self.json_data) self.apply_women_considerations_logic() # Hide factors based on women considerations + # Hide factors that have only a single indicator (indicator shown in their place) + self.model.toggle_single_child_factors_visibility(hide_single_child=True) self.treeView.expandAll() def export_json_to_file(self): @@ -861,18 +892,23 @@ def update_action_text(): animate_results_action.triggered.connect(self.animate_results) menu.addAction(animate_results_action) - add_geoe3_score = QAction("Add GeoE3 Score to Map") + add_geoe3_score = QAction("Add GeoE3 Score to Map (Raster)") add_geoe3_score.triggered.connect( lambda: add_to_map(item, key="result_file", layer_name="GeoE3 Score", group="GeoE3") ) menu.addAction(add_geoe3_score) + add_geoe3_score_grid = QAction("Add GeoE3 Score to Map (Grid)") + add_geoe3_score_grid.triggered.connect(lambda: add_grid_layer_to_map(item, "geoe3", self.working_directory)) + menu.addAction(add_geoe3_score_grid) + add_geoe3_by_population = QAction("Add GeoE3 by Pop to Map") add_geoe3_by_population.triggered.connect( - lambda: add_to_map( + lambda: add_grid_layer_to_map( item, - key="geoe3_by_population", - layer_name="GeoE3 by Population", + column_name="geoe3_by_population", + working_directory=self.working_directory, + layer_name="GeoE3 by Population (Grid)", group="GeoE3", ) ) @@ -923,6 +959,13 @@ def update_action_text(): add_factor_action = QAction("Add Factor", self) remove_dimension_action = QAction("Remove Dimension", self) + # Add grid layer action + add_grid_to_map_action = QAction("Add to map (Grid)", self) + column_name = f"dim_{item.attribute('id').lower().replace(' ', '_').replace('-', '_')}" + add_grid_to_map_action.triggered.connect( + lambda _, col=column_name, i=item: add_grid_layer_to_map(i, col, self.working_directory) + ) + # Connect actions add_factor_action.triggered.connect(lambda: self.model.add_factor(item)) remove_dimension_action.triggered.connect(lambda: self.model.remove_item(item)) @@ -932,6 +975,7 @@ def update_action_text(): menu.addAction(show_json_attributes_action) menu.addAction(clear_item_action) menu.addAction(add_to_map_action) + menu.addAction(add_grid_to_map_action) menu.addAction(run_item_action) menu.addAction(open_working_directory_action) menu.addAction(disable_action) @@ -942,6 +986,13 @@ def update_action_text(): add_indicator_action = QAction("Add Indicator", self) remove_factor_action = QAction("Remove Factor", self) + # Add grid layer action + add_grid_to_map_action = QAction("Add to map (Grid)", self) + column_name = f"fac_{item.attribute('id').lower().replace(' ', '_').replace('-', '_')}" + add_grid_to_map_action.triggered.connect( + lambda _, col=column_name, i=item: add_grid_layer_to_map(i, col, self.working_directory) + ) + # Connect actions edit_aggregation_action.triggered.connect(lambda: self.edit_factor_aggregation(item)) # Connect to method add_indicator_action.triggered.connect(lambda: self.model.add_indicator(item)) @@ -953,6 +1004,7 @@ def update_action_text(): menu.addAction(show_json_attributes_action) menu.addAction(clear_item_action) menu.addAction(add_to_map_action) + menu.addAction(add_grid_to_map_action) menu.addAction(run_item_action) menu.addAction(open_working_directory_action) menu.addAction(disable_action) @@ -963,14 +1015,24 @@ def update_action_text(): # of its parent factor... show_properties_action = QAction("🔘 Edit Weights", self) + # Add grid layer action (indicators use raw ID without prefix) + add_grid_to_map_action = QAction("Add to map (Grid)", self) + column_name = item.attribute("id").lower().replace(" ", "_").replace("-", "_") + add_grid_to_map_action.triggered.connect( + lambda _, col=column_name, i=item: add_grid_layer_to_map(i, col, self.working_directory) + ) + # Connect actions - show_properties_action.triggered.connect(lambda: self.edit_factor_aggregation(item.parent())) + show_properties_action.triggered.connect( + lambda: self.edit_factor_aggregation(item.parent(), selected_guids=[item.guid]) + ) # Add actions to menu menu = SolidMenu(self) menu.addAction(show_properties_action) menu.addAction(show_json_attributes_action) menu.addAction(clear_item_action) menu.addAction(add_to_map_action) + menu.addAction(add_grid_to_map_action) menu.addAction(run_item_action) menu.addAction(open_working_directory_action) menu.addAction(disable_action) @@ -1055,63 +1117,39 @@ def generate_study_area_report(self): self.overall_progress_bar.setVisible(False) def add_masked_scores_to_map(self, item): - """Add the masked scores to the map. + """Add the masked scores to the map as grid layers. Args: item: The analysis item containing masked score data. """ - add_to_map( + # Add GeoE3 masked score as grid layer + add_grid_layer_to_map( item, - key="geoe3_score_ghsl_masked_result_file", - layer_name="Masked GeoE3 Score", + column_name="geoe3_masked", + working_directory=self.working_directory, + layer_name="Masked GeoE3 Score (Grid)", group="GeoE3", ) - add_to_map( + # Add GeoE3 by population masked score as grid layer + add_grid_layer_to_map( item, - key="geoe3_by_population_by_opportunities_mask_result_file", - layer_name="Masked GeoE3 by Population Score", + column_name="geoe3_by_population_masked", + working_directory=self.working_directory, + layer_name="Masked GeoE3 by Population Score (Grid)", group="GeoE3", ) def add_opportunities_mask_to_map(self, item): - """Add the opportunities mask to the map with diagnostic feedback. + """Add the opportunities mask to the map as a grid layer. Args: item: The analysis item containing opportunities mask configuration. """ - mask_file = item.attribute("opportunities_mask_result_file") - - if not mask_file: - iface.messageBar().pushMessage( - "Opportunities Mask", - "Not configured. Run the opportunities mask processing first.", - level=Qgis.Warning, - duration=8, - ) - return - - if not os.path.exists(mask_file): - iface.messageBar().pushMessage( - "Opportunities Mask", - f"File not found: {os.path.basename(mask_file)}. Run the mask processing.", - level=Qgis.Warning, - duration=8, - ) - return - - if os.path.getsize(mask_file) == 0: - iface.messageBar().pushMessage( - "Opportunities Mask", - "File is empty (0 bytes). Run the mask processing again.", - level=Qgis.Warning, - duration=8, - ) - return - - add_to_map( + add_grid_layer_to_map( item, - key="opportunities_mask_result_file", - layer_name="Opportunities Mask", + column_name="opportunities_mask", + working_directory=self.working_directory, + layer_name="Opportunities Mask (Grid)", group="GeoE3", ) @@ -1631,17 +1669,24 @@ def edit_dimension_aggregation(self, dimension_item): dialog.saveWeightingsToModel() self.save_json_to_working_directory() # Save changes to the JSON if necessary - def edit_factor_aggregation(self, factor_item): + def edit_factor_aggregation(self, factor_item, selected_guids=None): """Open the FactorAggregationDialog for editing the weightings of layers in a factor. Args: factor_item: The factor item to edit. + selected_guids: Optional subset of indicator GUIDs to display in the dialog. """ factor_name = factor_item.data(0) factor_data = factor_item.attributes() if not factor_data: factor_data = {} - dialog = FactorAggregationDialog(factor_name, factor_data, factor_item, parent=self) + dialog = FactorAggregationDialog( + factor_name, + factor_data, + factor_item, + parent=self, + selected_guids=selected_guids, + ) if dialog.exec_(): # If OK was clicked dialog.save_weightings_to_model() self.save_json_to_working_directory() # Save changes to the JSON if necessary @@ -1713,6 +1758,49 @@ def analysis_scale(self): analysis_scale = self.model.get_analysis_item().attributes().get("analysis_scale", "national") return analysis_scale + def _get_study_area_area_km2(self) -> float: + """Get total study area area in km² from study_area_clip_polygons layer. + + Returns: + float: Total area in km². + """ + gpkg_path = os.path.join(self.working_directory, "study_area", "study_area.gpkg") + layer = QgsVectorLayer(f"{gpkg_path}|layername=study_area_clip_polygons", "study_area", "ogr") + if not layer.isValid(): + log_message( + f"Could not load study_area_clip_polygons from {gpkg_path}", + tag="GeoE3", + level=Qgis.Warning, + ) + return 0.0 + total_area = 0.0 + for feature in layer.getFeatures(): + geom = feature.geometry() + if geom: + total_area += geom.area() + return total_area / 1_000_000 + + def _get_render_strategy(self) -> str: + """Determine render strategy based on analysis scale and feature count. + + Uses feature count estimate to decide whether raster or vector rendering + is more appropriate for performance. + + Returns: + str: 'raster' or 'vector' + """ + + analysis_scale = self.analysis_scale() + + if analysis_scale == "regional": + return "vector" + + study_area_area_km2 = self._get_study_area_area_km2() + cell_size_m = self.cell_size_m() + estimated_features = (study_area_area_km2 * 1_000_000) / (cell_size_m**2) + + return "raster" if estimated_features > MAX_FEATURES_FOR_VECTOR else "vector" + def road_network_layer_path(self): """Get the layer used for network analysis. @@ -2040,7 +2128,22 @@ def on_workflow_completed(self, item, success): self.overall_progress_bar.setMaximum(self.items_to_run - 1) self.workflow_progress_bar.setValue(0) self.save_json_to_working_directory() - add_to_map(item) + # Add layer to map after workflow completes using auto-determined strategy + if item.role == "dimension": + column_name = f"dim_{item.attribute('id').lower().replace(' ', '_').replace('-', '_')}" + elif item.role == "factor": + column_name = f"fac_{item.attribute('id').lower().replace(' ', '_').replace('-', '_')}" + elif item.role == "indicator": + column_name = item.attribute("id").lower().replace(" ", "_").replace("-", "_") + elif item.role == "analysis": + column_name = "geoe3" # Analysis aggregation uses geoe3 column + else: + column_name = None + + if column_name is None or self._get_render_strategy() == "raster": + add_to_map(item) + else: + add_grid_layer_to_map(item, column_name, self.working_directory) # Now cancel the animated icon node_index = self.model.itemIndex(item) diff --git a/geest/gui/views/treeview.py b/geest/gui/views/treeview.py index dfb2eb8c..fff890ab 100644 --- a/geest/gui/views/treeview.py +++ b/geest/gui/views/treeview.py @@ -321,6 +321,53 @@ def toggle_indicator_visibility(self, visible: bool, parent_item=None): # Notify view about layout changes self.layoutChanged.emit() + def toggle_single_child_factors_visibility(self, hide_single_child: bool): + """ + Toggles the visibility of factors that have only one child indicator. + + When hide_single_child is True, factors with exactly one child indicator + are hidden, and their indicator is promoted to appear directly under + the dimension. + + Args: + hide_single_child (bool): If True, hide factors with only one child. + If False, show all factors. + """ + analysis_item = self.get_analysis_item() + if not analysis_item: + return + + for dimension in analysis_item.childItems: + for factor in dimension.childItems: + # Check if factor has exactly one child indicator + if len(factor.childItems) == 1: + # Hide the factor if hide_single_child is True + factor.set_visibility(not hide_single_child) + else: + # Always show factors with multiple children + factor.set_visibility(True) + + # Notify view about layout changes + self.layoutChanged.emit() + + def are_single_child_factors_hidden(self) -> bool: + """ + Check if single-child factors are currently hidden. + + Returns: + bool: True if any single-child factor is hidden, False otherwise. + """ + analysis_item = self.get_analysis_item() + if not analysis_item: + return False + + for dimension in analysis_item.childItems: + for factor in dimension.childItems: + if len(factor.childItems) == 1: + # Return visibility state of first single-child factor found + return not factor.is_visible() + return False + def data(self, index, role): """ Provides data for the given index and role, including displaying custom attributes such as the font color, @@ -639,21 +686,25 @@ def remove_item(self, item): def rowCount(self, parent=QModelIndex()): """ - Returns the number of child items for the given parent, excluding hidden items if visibility is off. + Returns the number of effective visible children for the given parent. + + This uses get_effective_visible_children() which handles: + - Normal visible children + - Promotion of grandchildren when a child is hidden but has exactly one child Args: parent (QModelIndex): The parent index. Returns: - int: The number of visible child items under the parent. + int: The number of effective visible child items under the parent. """ if not parent.isValid(): parentItem = self.rootItem else: parentItem = parent.internalPointer() - # Count only visible items - return len([child for child in parentItem.childItems if child.is_visible()]) + # Use effective visible children (handles hidden single-child factors) + return len(parentItem.get_effective_visible_children()) def columnCount(self, parent=QModelIndex()): """ @@ -691,36 +742,51 @@ def guidIndex(self, guid): """ return self._findIndexByGuid(self.rootItem, guid) - def _findIndexByGuid(self, parent_item, target_guid, parent_index=QModelIndex()): + def _findIndexByGuid(self, parent_item, target_guid): """ Recursively searches for the target guid within the children of the given parent item. + Uses get_effective_visible_children() to calculate the correct visual row + position, accounting for hidden single-child factors. + Args: parent_item (JsonTreeItem): The parent item to start searching from. target_guid (str): The GUID of the item to search for. - parent_index (QModelIndex): The QModelIndex of the parent item. Returns: QModelIndex: The QModelIndex of the target item, or an invalid QModelIndex if not found. """ - for row in range(parent_item.childCount()): - child_item = parent_item.child(row) + # Get effective visible children for correct row calculation + effective_children = parent_item.get_effective_visible_children() - # If the item's UUID matches, return its QModelIndex + for row, child_item in enumerate(effective_children): + # If the item's GUID matches, return its QModelIndex if child_item.guid == target_guid: return self.createIndex(row, 0, child_item) # Recursively search children - child_index = self._findIndexByGuid(child_item, target_guid, self.createIndex(row, 0, parent_item)) + child_index = self._findIndexByGuid(child_item, target_guid) if child_index.isValid(): return child_index + # Also search in hidden children (in case the item is under a hidden factor) + for child_item in parent_item.childItems: + if not child_item.is_visible(): + # Search within hidden item's children + child_index = self._findIndexByGuid(child_item, target_guid) + if child_index.isValid(): + return child_index + return QModelIndex() # Return invalid QModelIndex if not found def index(self, row, column, parent=QModelIndex()): """ Creates a QModelIndex for the specified row and column under the given parent. + Uses get_effective_visible_children() to handle: + - Normal visible children + - Promotion of grandchildren when a child is hidden but has exactly one child + Args: row (int): The row of the child item. column (int): The column of the child item. @@ -737,8 +803,10 @@ def index(self, row, column, parent=QModelIndex()): else: parentItem = parent.internalPointer() - childItem = parentItem.child(row) - if childItem: + # Use effective visible children (handles hidden single-child factors) + effective_children = parentItem.get_effective_visible_children() + if row < len(effective_children): + childItem = effective_children[row] return self.createIndex(row, column, childItem) return QModelIndex() @@ -746,6 +814,10 @@ def parent(self, index): """ Returns the parent index of the specified index. + Uses get_visual_parent() to handle the case where the actual parent + is hidden (e.g., a factor with only one child). In this case, the + visual parent is the grandparent (dimension). + Args: index (QModelIndex): The child index. @@ -756,12 +828,25 @@ def parent(self, index): return QModelIndex() childItem = index.internalPointer() - parentItem = childItem.parent() + # Use visual parent which skips hidden parents + parentItem = childItem.get_visual_parent() - if parentItem == self.rootItem: + if parentItem is None or parentItem == self.rootItem: return QModelIndex() - return self.createIndex(parentItem.row(), 0, parentItem) + # Calculate the visible row position of the parent + # We need to find the parent's position in its own parent's effective children + grandparent = parentItem.get_visual_parent() + if grandparent is None: + grandparent = self.rootItem + + effective_siblings = grandparent.get_effective_visible_children() + if parentItem in effective_siblings: + row = effective_siblings.index(parentItem) + else: + row = 0 + + return self.createIndex(row, 0, parentItem) def headerData(self, section, orientation, role=Qt.ItemDataRole.DisplayRole): """ @@ -855,20 +940,11 @@ def collapse_node_in_view(self, item): self.treeView.setExpanded(index, False) def toggle_only_child_indicator_nodes(self): - """Toggles visibility of indicator nodes if it has no siblings.""" - indicators_visible = self._indicators_only_child() - self.model().toggle_indicator_visibility(not indicators_visible) + """Toggles visibility of factors that have only one child indicator. - def _indicators_only_child(self): - """Check if indicators are currently the only child under their parent. - - Returns: - bool: True if indicators are only children, False otherwise. + When enabled, single-child factors are hidden and their indicator + is shown directly under the dimension. """ model = self.model() - analysis_item = model.get_analysis_item() - for dimension in analysis_item.childItems: - for factor in dimension.childItems: - for indicator in factor.childItems: - return indicator.is_only_child() - return True + currently_hidden = model.are_single_child_factors_hidden() + model.toggle_single_child_factors_visibility(not currently_hidden) diff --git a/geest/gui/widgets/configuration_widgets/base_configuration_widget.py b/geest/gui/widgets/configuration_widgets/base_configuration_widget.py index 491981e5..09bb3b5b 100644 --- a/geest/gui/widgets/configuration_widgets/base_configuration_widget.py +++ b/geest/gui/widgets/configuration_widgets/base_configuration_widget.py @@ -43,6 +43,7 @@ def __init__( self.analysis_mode = analysis_mode if not humanised_label: humanised_label = analysis_mode.replace("_", " ").title() + humanised_label = humanised_label.replace("Ghsl", "GHSL") self.attributes = attributes # Main layout diff --git a/geest/gui/widgets/configuration_widgets/safety_polygon_configuration_widget.py b/geest/gui/widgets/configuration_widgets/safety_polygon_configuration_widget.py index afa55414..cc926556 100644 --- a/geest/gui/widgets/configuration_widgets/safety_polygon_configuration_widget.py +++ b/geest/gui/widgets/configuration_widgets/safety_polygon_configuration_widget.py @@ -6,9 +6,12 @@ from qgis.core import Qgis from qgis.PyQt.QtWidgets import ( + QAbstractSpinBox, + QHeaderView, QLabel, QSizePolicy, QSpinBox, + QStyleFactory, QTableWidget, QTableWidgetItem, ) @@ -35,6 +38,16 @@ class SafetyPolygonConfigurationWidget(BaseConfigurationWidget): polygon_shapefile_line_edit (QLineEdit): Line edit for entering/selecting a polygon layer shapefile. """ + def _set_spinbox_style(self, spin_box: QSpinBox, warning: bool) -> None: + """Apply minimal styling while preserving native arrow rendering.""" + text_color = "#d11" if warning else "#000" + fusion_style = QStyleFactory.create("Fusion") + if fusion_style is not None: + spin_box.setStyle(fusion_style) + spin_box.setButtonSymbols(QAbstractSpinBox.UpDownArrows) + if spin_box.lineEdit() is not None: + spin_box.lineEdit().setStyleSheet(f"color: {text_color};") + def add_internal_widgets(self) -> None: """ Adds the internal widgets required for selecting polygon layers and their corresponding shapefiles. @@ -47,11 +60,17 @@ def add_internal_widgets(self) -> None: self.table_widget.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding) # Stop the label being editable self.table_widget.setEditTriggers(QTableWidget.NoEditTriggers) + self.table_widget.verticalHeader().setVisible(True) + self.table_widget.verticalHeader().setDefaultSectionSize(24) + self.table_widget.verticalHeader().setMinimumSectionSize(20) + self.table_widget.verticalHeader().setSectionResizeMode(QHeaderView.Fixed) + self.table_widget.verticalHeader().setFixedWidth(28) self.internal_layout.addWidget(self.table_widget) self.table_widget.setColumnCount(2) - self.table_widget.setColumnWidth(1, 80) + self.table_widget.setColumnWidth(1, 110) self.table_widget.horizontalHeader().setStretchLastSection(False) - self.table_widget.horizontalHeader().setSectionResizeMode(0, self.table_widget.horizontalHeader().Stretch) + self.table_widget.horizontalHeader().setSectionResizeMode(0, QHeaderView.Stretch) + self.table_widget.horizontalHeader().setSectionResizeMode(1, QHeaderView.ResizeToContents) return self.populate_table() self.internal_layout.addWidget(self.table_widget) @@ -105,9 +124,10 @@ def validate_value(value): self.table_widget.setItem(row, 0, name_item) value_item.setRange(0, 100) # Set spinner range value_item.setValue(value) # Default value + self._set_spinbox_style(value_item, warning=False) self.table_widget.setCellWidget(row, 1, value_item) - def on_value_changed(value): + def on_value_changed(value, spin_box=value_item): """🔄 On value changed. Args: @@ -115,10 +135,10 @@ def on_value_changed(value): """ # Color handling for current cell if value is None or not (0 <= value <= 100): - value_item.setStyleSheet("color: red;") - value_item.setValue(0) + self._set_spinbox_style(spin_box, warning=True) + spin_box.setValue(0) else: - value_item.setStyleSheet("color: black;") + self._set_spinbox_style(spin_box, warning=False) self.update_cell_colors() self.update_data() @@ -141,7 +161,7 @@ def update_cell_colors(self): for r in range(self.table_widget.rowCount()): spin_widget = self.table_widget.cellWidget(r, 1) if spin_widget: - spin_widget.setStyleSheet("color: red;" if all_zeros else "color: black;") + self._set_spinbox_style(spin_widget, warning=all_zeros) def table_to_dict(self): """⚙️ Table to dict. diff --git a/geest/gui/widgets/datasource_widgets/__init__.py b/geest/gui/widgets/datasource_widgets/__init__.py index 54d944b3..82d08b25 100644 --- a/geest/gui/widgets/datasource_widgets/__init__.py +++ b/geest/gui/widgets/datasource_widgets/__init__.py @@ -10,9 +10,16 @@ from .acled_csv_datasource_widget import AcledCsvDataSourceWidget # noqa F401 from .base_datasource_widget import BaseDataSourceWidget # noqa F401 from .csv_datasource_widget import CsvDataSourceWidget # noqa F401 +from .download_task_controls import DownloadTaskControls # noqa F401 from .eplex_datasource_widget import EPLEXDataSourceWidget # noqa F401 from .fixed_value_datasource_widget import FixedValueDataSourceWidget # noqa F401 from .raster_datasource_widget import RasterDataSourceWidget # noqa F401 +from .s2s_datasource_widget import S2SDataSourceWidget # noqa F401 +from .s2s_education_datasource_widget import S2SEducationDataSourceWidget # noqa F401 +from .s2s_environmental_hazards_raster_datasource_widget import ( # noqa F401 + S2SEnvironmentalHazardsRasterDataSourceWidget, +) +from .s2s_ntl_raster_datasource_widget import S2SNTLRasterDataSourceWidget # noqa F401 from .vector_and_field_datasource_widget import ( # noqa F401 VectorAndFieldDataSourceWidget, ) diff --git a/geest/gui/widgets/datasource_widgets/download_task_controls.py b/geest/gui/widgets/datasource_widgets/download_task_controls.py new file mode 100644 index 00000000..23899b28 --- /dev/null +++ b/geest/gui/widgets/datasource_widgets/download_task_controls.py @@ -0,0 +1,157 @@ +# -*- coding: utf-8 -*- +"""Reusable download button controls with spinner and lifecycle states.""" + +from qgis.PyQt.QtCore import QSize, Qt, QTimer +from qgis.PyQt.QtGui import QMovie +from qgis.PyQt.QtWidgets import QHBoxLayout, QLabel, QPushButton, QSizePolicy, QWidget + +from geest.utilities import resources_path + + +class DownloadTaskControls: + """Encapsulate common download button + spinner behavior for datasource widgets.""" + + def __init__(self, button_text: str, tooltip: str, click_handler): + """Create reusable controls and wire click callback. + + Args: + button_text: Default button text. + tooltip: Default button tooltip. + click_handler: Callback invoked on button click. + """ + self.default_text = button_text + self.default_tooltip = tooltip + self.default_style = "padding: 5px 10px;" + + self.container = QWidget() + self.container.setSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Preferred) + layout = QHBoxLayout(self.container) + layout.setContentsMargins(0, 0, 0, 0) + layout.setSpacing(6) + + self.button = QPushButton(self.default_text) + self.button.setSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Fixed) + self.button.setToolTip(self.default_tooltip) + self.button.setStyleSheet(self.default_style) + self.button.clicked.connect(click_handler) + + self.spinner_label = QLabel() + self.spinner_movie = QMovie(resources_path("resources", "throbber.gif")) + self.spinner_movie.setScaledSize(QSize(24, 24)) + self.spinner_label.setMovie(self.spinner_movie) + self.spinner_label.setVisible(False) + self.spinner_label.setAlignment(Qt.AlignVCenter) + + layout.addWidget(self.button) + layout.addWidget(self.spinner_label) + layout.addStretch() + + def set_running(self) -> None: + """Set button to downloading state and show spinner.""" + self._set_state( + text="Downloading...", + enabled=False, + style=self.default_style, + tooltip=self.default_tooltip, + ) + self.spinner_label.setVisible(True) + self.spinner_movie.start() + + def update_progress(self, message: str) -> None: + """Update button text based on task progress message.""" + if "Processing" in message: + self.button.setText("Processing...") + elif "complete" in message.lower(): + self.button.setText("Complete!") + + def set_downloaded(self, reset_after_ms: int = 2000) -> None: + """Set successful completion state and auto-reset.""" + self._set_state( + text="Downloaded!", + enabled=True, + style="background-color: #ccffcc; padding: 5px 10px;", + tooltip=self.default_tooltip, + stop_spinner=True, + ) + QTimer.singleShot(reset_after_ms, self.reset) + + def set_download_failed(self, error_message: str) -> None: + """Set standard failed state with retry tooltip.""" + self._set_state( + text="Download Failed!", + enabled=True, + style="background-color: #ffcccc; padding: 5px 10px;", + tooltip=f"Error: {error_message}\n\nClick to retry.", + stop_spinner=True, + ) + + def set_error(self, error_message: str) -> None: + """Set startup/runtime error state.""" + self._set_state( + text="Error!", + enabled=True, + style="background-color: #ffcccc; padding: 5px 10px;", + tooltip=f"Error: {error_message}", + stop_spinner=True, + ) + + def set_not_found(self, path: str) -> None: + """Set missing-output state.""" + self._set_state( + text="Not Found!", + enabled=True, + style="background-color: #ffcccc; padding: 5px 10px;", + tooltip=f"Error: Output file not found: {path}", + stop_spinner=True, + ) + + def set_load_failed(self, path: str) -> None: + """Set invalid-output load failure state.""" + self._set_state( + text="Load Failed!", + enabled=True, + style="background-color: #ffcccc; padding: 5px 10px;", + tooltip=f"Error: Could not load output layer: {path}", + stop_spinner=True, + ) + + def set_cancelled(self) -> None: + """Set cancelled state.""" + self._set_state( + text="Cancelled", + enabled=True, + style="background-color: #ffffcc; padding: 5px 10px;", + tooltip="Download was cancelled. Click to retry.", + stop_spinner=True, + ) + + def stop_spinner(self) -> None: + """Stop spinner animation and hide indicator.""" + self.spinner_movie.stop() + self.spinner_label.setVisible(False) + + def reset(self) -> None: + """Reset button to initial state.""" + self._set_state( + text=self.default_text, + enabled=True, + style=self.default_style, + tooltip=self.default_tooltip, + stop_spinner=True, + ) + + def _set_state( + self, + text: str, + enabled: bool, + style: str, + tooltip: str, + stop_spinner: bool = False, + ) -> None: + """Apply a full UI state for the button controls.""" + self.button.setText(text) + self.button.setEnabled(enabled) + self.button.setStyleSheet(style) + self.button.setToolTip(tooltip) + if stop_spinner: + self.stop_spinner() diff --git a/geest/gui/widgets/datasource_widgets/s2s_datasource_widget.py b/geest/gui/widgets/datasource_widgets/s2s_datasource_widget.py new file mode 100644 index 00000000..f96c3a56 --- /dev/null +++ b/geest/gui/widgets/datasource_widgets/s2s_datasource_widget.py @@ -0,0 +1,314 @@ +# -*- coding: utf-8 -*- +"""Space2Stats datasource widget.""" + +import json +import os +from typing import List, Optional + +from qgis.core import ( + QgsApplication, + QgsCoordinateReferenceSystem, + QgsCoordinateTransform, + QgsGeometry, + QgsProject, + QgsVectorLayer, +) +from qgis.PyQt.QtCore import QSettings +from qgis.PyQt.QtWidgets import QLabel, QLineEdit, QMessageBox, QSizePolicy + +from geest.core import S2STaskGate +from geest.core.tasks import S2SDownloaderTask + +from .download_task_controls import DownloadTaskControls +from .vector_datasource_widget import VectorDataSourceWidget + + +class S2SDataSourceWidget(VectorDataSourceWidget): + """Vector datasource widget with optional Space2Stats download support.""" + + def add_internal_widgets(self) -> None: + """Build base vector controls and append S2S controls.""" + super().add_internal_widgets() + + self.s2s_fields_line_edit = QLineEdit() + self.s2s_fields_line_edit.setPlaceholderText("S2S fields (comma separated)") + self.s2s_fields_line_edit.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed) + initial_fields = self.attributes.get("s2s_fields", []) + if isinstance(initial_fields, list) and initial_fields: + self.s2s_fields_line_edit.setText(",".join(str(field) for field in initial_fields)) + elif isinstance(self.attributes.get("s2s_field"), str): + self.s2s_fields_line_edit.setText(self.attributes.get("s2s_field")) + self.s2s_fields_line_edit.textChanged.connect(self.update_attributes) + self.layout.addWidget(self.s2s_fields_line_edit) + + self.s2s_controls = DownloadTaskControls( + button_text="Download from S2S", + tooltip="Download data from Space2Stats", + click_handler=self.fetch_from_s2s, + ) + self.s2s_fetch_button = self.s2s_controls.button + self.layout.addWidget(self.s2s_controls.container) + + self.s2s_status_label = QLabel() + self.s2s_status_label.setSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Fixed) + self.s2s_status_label.setMinimumWidth(90) + self.s2s_status_label.setMaximumWidth(170) + self.layout.addWidget(self.s2s_status_label) + + self.layout.setStretchFactor(self.layer_combo, 4) + self.layout.setStretchFactor(self.shapefile_line_edit, 4) + self.layout.setStretchFactor(self.s2s_fields_line_edit, 3) + + self._s2s_error_handled = False + self.s2s_task = None + self._s2s_gate_token = None + self.s2s_output_path = self.attributes.get("s2s_output_path", "") + self._load_existing_s2s_output() + + def fetch_from_s2s(self) -> None: + """Start a background task to fetch S2S data for the study area.""" + settings = QSettings() + working_directory = settings.value("last_working_directory", "") + if not working_directory or not os.path.exists(working_directory): + QMessageBox.warning( + self, + "No Working Directory", + "No valid working directory found. Please create or open a project first.", + ) + return + + study_area_gpkg = os.path.join(working_directory, "study_area", "study_area.gpkg") + if not os.path.exists(study_area_gpkg): + QMessageBox.warning( + self, + "Study Area Required", + "Study area GeoPackage not found. Please create a project first.", + ) + return + + fields = self._parse_fields(self.s2s_fields_line_edit.text()) + if not fields: + QMessageBox.warning( + self, + "S2S Fields Required", + "Please enter at least one S2S field (comma separated).", + ) + return + + aoi_layer = QgsVectorLayer(f"{study_area_gpkg}|layername=study_area_bboxes", "study_area_bboxes", "ogr") + if not aoi_layer.isValid() or aoi_layer.featureCount() == 0: + QMessageBox.warning( + self, + "Invalid Study Area", + "Could not load study_area_bboxes from study_area.gpkg.", + ) + return + + aoi_feature = self._build_aoi_feature(aoi_layer) + if not aoi_feature: + QMessageBox.warning( + self, + "Invalid AOI", + "Failed to build AOI feature from study area geometry.", + ) + return + + self.s2s_output_path = os.path.join(working_directory, "study_area", f"s2s_{self.widget_key}.gpkg") + + gate_label = f"widget:{self.widget_key}" + token = S2STaskGate.acquire(gate_label) + if not token: + active = S2STaskGate.active_label() or "another panel" + QMessageBox.information( + self, + "S2S Busy", + f"Another S2S download is currently running ({active}). Please wait for it to finish.", + ) + return + self._s2s_gate_token = token + + self.s2s_controls.set_running() + self.s2s_status_label.setText("Fetching S2S data...") + + self._s2s_error_handled = False + self.s2s_task = S2SDownloaderTask( + aoi=aoi_feature, + fields=fields, + working_dir=working_directory, + filename=f"s2s_{self.widget_key}", + spatial_join_method="centroid", + geometry="point", + delete_existing=True, + ) + + self.s2s_task.progress_updated.connect(self._on_s2s_progress) + self.s2s_task.error_occurred.connect(self._on_s2s_error) + self.s2s_task.taskCompleted.connect(self._on_s2s_completed) + self.s2s_task.taskTerminated.connect(self._on_s2s_terminated) + QgsApplication.taskManager().addTask(self.s2s_task) + + def _on_s2s_progress(self, message: str) -> None: + """Update S2S status text from task progress.""" + self.s2s_status_label.setText(message) + self.s2s_controls.update_progress(message) + + def _on_s2s_error(self, message: str) -> None: + """Handle S2S task errors.""" + self._s2s_error_handled = True + self._release_s2s_gate() + self.s2s_status_label.setText("S2S download failed") + self.s2s_controls.set_download_failed(message) + friendly_message = self._humanize_s2s_error(message) + QMessageBox.warning(self, "S2S Download Failed", friendly_message) + + def _on_s2s_terminated(self) -> None: + """Handle cancelled/terminated S2S tasks.""" + self._release_s2s_gate() + if self._s2s_error_handled: + return + self.s2s_status_label.setText("S2S task terminated") + self.s2s_controls.set_cancelled() + + def _on_s2s_completed(self) -> None: + """Load output layer after successful S2S task completion.""" + self._release_s2s_gate() + self.s2s_controls.reset() + + if not self.s2s_output_path or not os.path.exists(self.s2s_output_path): + self.s2s_status_label.setText("S2S output not found") + self.s2s_controls.set_not_found(self.s2s_output_path) + return + + layer_name = os.path.splitext(os.path.basename(self.s2s_output_path))[0] + output_layer = self._load_or_reuse_vector_layer(self.s2s_output_path, layer_name) + if output_layer is None: + self.s2s_status_label.setText("S2S output invalid") + self.s2s_controls.set_load_failed(self.s2s_output_path) + QMessageBox.warning(self, "Invalid S2S Output", "S2S output file exists but could not be loaded.") + return + + self._switch_to_layer_mode(output_layer) + self.s2s_status_label.setText("S2S download complete") + self.s2s_controls.set_downloaded() + self.update_attributes() + + def _release_s2s_gate(self) -> None: + """Release global S2S gate lock for this widget task.""" + if getattr(self, "_s2s_gate_token", None): + S2STaskGate.release(self._s2s_gate_token) + self._s2s_gate_token = None + + def _load_existing_s2s_output(self) -> None: + """Auto-select existing S2S output when available.""" + if not self.s2s_output_path or not os.path.exists(self.s2s_output_path): + return + + layer_name = os.path.splitext(os.path.basename(self.s2s_output_path))[0] + output_layer = self._load_or_reuse_vector_layer(self.s2s_output_path, layer_name) + if output_layer is None: + self.s2s_status_label.setText("S2S output invalid") + return + + self._switch_to_layer_mode(output_layer) + self.s2s_controls.set_downloaded() + + def _switch_to_layer_mode(self, output_layer: QgsVectorLayer) -> None: + """Select downloaded layer and reset manual path input mode.""" + self.shapefile_line_edit.clear() + self.shapefile_line_edit.setVisible(False) + self.layer_combo.setVisible(True) + self.layer_combo.setLayer(output_layer) + + @staticmethod + def _load_or_reuse_vector_layer(layer_path: str, layer_name: str) -> Optional[QgsVectorLayer]: + """Load a vector layer once, reusing an existing project layer when possible.""" + target_path = os.path.normpath(os.path.abspath(layer_path)) + for existing_layer in QgsProject.instance().mapLayers().values(): + if not isinstance(existing_layer, QgsVectorLayer): + continue + source = existing_layer.source() or "" + source_path = os.path.normpath(os.path.abspath(source.split("|")[0])) + if source_path != target_path: + continue + return existing_layer + + output_layer = QgsVectorLayer(f"{layer_path}|layername={layer_name}", layer_name, "ogr") + if not output_layer.isValid(): + output_layer = QgsVectorLayer(layer_path, layer_name, "ogr") + if not output_layer.isValid(): + return None + + QgsProject.instance().addMapLayer(output_layer) + return output_layer + + def update_attributes(self): + """Update base layer attributes and S2S metadata attributes.""" + super().update_attributes() + if not hasattr(self, "s2s_fields_line_edit"): + return + fields = self._parse_fields(self.s2s_fields_line_edit.text()) + self.attributes["s2s_fields"] = fields + self.attributes["s2s_fields_text"] = self.s2s_fields_line_edit.text() + self.attributes["s2s_spatial_join_method"] = "centroid" + if self.s2s_output_path: + self.attributes["s2s_output_path"] = self.s2s_output_path + + @staticmethod + def _parse_fields(raw_text: str) -> List[str]: + """Parse comma-separated field names into a de-duplicated list.""" + fields = [token.strip() for token in raw_text.split(",") if token.strip()] + unique_fields = [] + for field in fields: + if field not in unique_fields: + unique_fields.append(field) + return unique_fields + + @staticmethod + def _build_aoi_feature(layer: QgsVectorLayer) -> dict: + """Build a GeoJSON feature from AOI geometry in EPSG:4326.""" + geometries = [] + source_crs = layer.crs() + target_crs = QgsCoordinateReferenceSystem("EPSG:4326") + transform = None + + if source_crs.isValid() and source_crs != target_crs: + transform = QgsCoordinateTransform(source_crs, target_crs, QgsProject.instance()) + + for feature in layer.getFeatures(): + geometry = feature.geometry() + if not geometry or geometry.isEmpty(): + continue + transformed_geometry = QgsGeometry(geometry) + if transform is not None: + transformed_geometry.transform(transform) + geometries.append(transformed_geometry) + + if not geometries: + return {} + + union_geometry = QgsGeometry.unaryUnion(geometries) + if not union_geometry or union_geometry.isEmpty(): + return {} + + return { + "type": "Feature", + "geometry": json.loads(union_geometry.asJson()), + "properties": {}, + } + + @staticmethod + def _humanize_s2s_error(message: str) -> str: + """Convert low-level S2S errors into user-friendly text.""" + lowered = str(message).lower() + if "503" in lowered or "service temporarily unavailable" in lowered or "server error (503)" in lowered: + return ( + "The Space2Stats service is temporarily unavailable (503). " "Please wait a few minutes and try again." + ) + if "exterior must be valid" in lowered or "coordinate" in lowered: + return ( + "The study area geometry sent to S2S is invalid in WGS84 coordinates. " + "Please recreate or repair the study area and try again." + ) + if "fields are unavailable" in lowered: + return "The selected S2S field is unavailable. Please refresh available fields and try again." + return message diff --git a/geest/gui/widgets/datasource_widgets/s2s_education_datasource_widget.py b/geest/gui/widgets/datasource_widgets/s2s_education_datasource_widget.py new file mode 100644 index 00000000..43b33b4e --- /dev/null +++ b/geest/gui/widgets/datasource_widgets/s2s_education_datasource_widget.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- +"""S2S-backed Education datasource widget.""" + +import os + +from qgis.core import QgsApplication, QgsMapLayerProxyModel, QgsVectorLayer +from qgis.PyQt.QtCore import QSettings +from qgis.PyQt.QtWidgets import QMessageBox + +from geest.core import S2STaskGate +from geest.core.constants import DEFAULT_S2S_EDUCATION_URBANIZATION_FIELDS +from geest.core.tasks import S2SDownloaderTask + +from .s2s_datasource_widget import S2SDataSourceWidget + + +class S2SEducationDataSourceWidget(S2SDataSourceWidget): + """Education-specific S2S datasource widget with fixed field configuration.""" + + OUTPUT_FILENAME = "s2s_education" + + def add_internal_widgets(self) -> None: + """Build controls and hide manual S2S fields input for Education.""" + super().add_internal_widgets() + if hasattr(self, "layer_combo"): + self.layer_combo.setFilters(QgsMapLayerProxyModel.PointLayer | QgsMapLayerProxyModel.PolygonLayer) + default_fields_text = ",".join(DEFAULT_S2S_EDUCATION_URBANIZATION_FIELDS) + self.s2s_fields_line_edit.setText(default_fields_text) + self.s2s_fields_line_edit.setEnabled(False) + self.s2s_fields_line_edit.setVisible(False) + + settings = QSettings() + working_directory = settings.value("last_working_directory", "") + self._load_best_available_s2s_output(working_directory) + + def _load_best_available_s2s_output(self, working_directory: str) -> None: + """Load configured output path or fallback to default Education output path.""" + configured_path = str(self.s2s_output_path or "").strip() + candidate_paths = [] + if configured_path: + candidate_paths.append(configured_path) + + fallback_path = self._resolve_default_s2s_output_path(working_directory) + if fallback_path and fallback_path not in candidate_paths: + candidate_paths.append(fallback_path) + + for candidate in candidate_paths: + if not candidate or not os.path.exists(candidate): + continue + + layer_name = os.path.splitext(os.path.basename(candidate))[0] + output_layer = self._load_or_reuse_vector_layer(candidate, layer_name) + if output_layer is None: + continue + + self.s2s_output_path = candidate + self._switch_to_layer_mode(output_layer) + self.s2s_controls.set_downloaded() + self.update_attributes() + return + + @staticmethod + def _resolve_default_s2s_output_path(working_directory: str) -> str: + """Resolve the standard Education S2S output path.""" + if not working_directory: + return "" + return os.path.join(working_directory, "study_area", f"{S2SEducationDataSourceWidget.OUTPUT_FILENAME}.gpkg") + + def fetch_from_s2s(self) -> None: + """Fetch Education S2S dataset using fixed output and field configuration.""" + settings = QSettings() + working_directory = settings.value("last_working_directory", "") + if not working_directory or not os.path.exists(working_directory): + QMessageBox.warning( + self, + "No Working Directory", + "No valid working directory found. Please create or open a project first.", + ) + return + + study_area_gpkg = os.path.join(working_directory, "study_area", "study_area.gpkg") + if not os.path.exists(study_area_gpkg): + QMessageBox.warning( + self, + "Study Area Required", + "Study area GeoPackage not found. Please create a project first.", + ) + return + + aoi_layer = QgsVectorLayer(f"{study_area_gpkg}|layername=study_area_bboxes", "study_area_bboxes", "ogr") + if not aoi_layer.isValid() or aoi_layer.featureCount() == 0: + QMessageBox.warning( + self, + "Invalid Study Area", + "Could not load study_area_bboxes from study_area.gpkg.", + ) + return + + aoi_feature = self._build_aoi_feature(aoi_layer) + if not aoi_feature: + QMessageBox.warning( + self, + "Invalid AOI", + "Failed to build AOI feature from study area geometry.", + ) + return + + fields = list(DEFAULT_S2S_EDUCATION_URBANIZATION_FIELDS) + self.s2s_output_path = os.path.join(working_directory, "study_area", f"{self.OUTPUT_FILENAME}.gpkg") + + gate_label = "widget:education" + token = S2STaskGate.acquire(gate_label) + if not token: + active = S2STaskGate.active_label() or "another panel" + QMessageBox.information( + self, + "S2S Busy", + f"Another S2S download is currently running ({active}). Please wait for it to finish.", + ) + return + self._s2s_gate_token = token + + self.s2s_controls.set_running() + self.s2s_status_label.setText("Fetching S2S data...") + + self._s2s_error_handled = False + self.s2s_task = S2SDownloaderTask( + aoi=aoi_feature, + fields=fields, + working_dir=working_directory, + filename=self.OUTPUT_FILENAME, + spatial_join_method="centroid", + geometry="point", + delete_existing=True, + ) + + self.s2s_task.progress_updated.connect(self._on_s2s_progress) + self.s2s_task.error_occurred.connect(self._on_s2s_error) + self.s2s_task.taskCompleted.connect(self._on_s2s_completed) + self.s2s_task.taskTerminated.connect(self._on_s2s_terminated) + QgsApplication.taskManager().addTask(self.s2s_task) + + def update_attributes(self): + """Persist fixed Education S2S fields and common metadata.""" + super().update_attributes() + self.attributes["s2s_fields"] = list(DEFAULT_S2S_EDUCATION_URBANIZATION_FIELDS) + self.attributes["s2s_fields_text"] = ",".join(DEFAULT_S2S_EDUCATION_URBANIZATION_FIELDS) diff --git a/geest/gui/widgets/datasource_widgets/s2s_environmental_hazards_raster_datasource_widget.py b/geest/gui/widgets/datasource_widgets/s2s_environmental_hazards_raster_datasource_widget.py new file mode 100644 index 00000000..0c9c6ece --- /dev/null +++ b/geest/gui/widgets/datasource_widgets/s2s_environmental_hazards_raster_datasource_widget.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- +"""S2S-backed Environmental Hazards raster datasource widget.""" + +import os + +from qgis.core import QgsApplication, QgsVectorLayer +from qgis.PyQt.QtCore import QSettings +from qgis.PyQt.QtWidgets import QFileDialog, QMessageBox + +from geest.core import S2STaskGate +from geest.core.constants import DEFAULT_S2S_ENV_HAZARD_FIELDS +from geest.core.tasks import S2SDownloaderTask + +from .s2s_datasource_widget import S2SDataSourceWidget +from .s2s_ntl_raster_datasource_widget import S2SNTLRasterDataSourceWidget + + +class S2SEnvironmentalHazardsRasterDataSourceWidget(S2SNTLRasterDataSourceWidget): + """Regional datasource widget that fetches hazard values from S2S.""" + + def add_internal_widgets(self) -> None: + """Build controls and configure hazard-specific S2S defaults.""" + super().add_internal_widgets() + self.s2s_vector_field_combo.setLayer(None) + self.s2s_vector_field_combo.setCurrentIndex(-1) + self.s2s_vector_field_combo.setEnabled(False) + self.s2s_vector_field_combo.setVisible(False) + self.s2s_ntl_field = self._hazard_field_from_attributes() + self.s2s_status_label.setToolTip(f"S2S field: {self.s2s_ntl_field}") + self._select_existing_hazard_output_layer() + + def _update_vector_field_combo(self) -> None: + """Disable manual field selection for S2S-specific hazards workflow.""" + if hasattr(self, "s2s_vector_field_combo"): + self.s2s_vector_field_combo.setLayer(None) + self.s2s_vector_field_combo.setCurrentIndex(-1) + self.s2s_vector_field_combo.setEnabled(False) + self.s2s_vector_field_combo.setVisible(False) + + def fetch_from_s2s(self) -> None: + """Fetch S2S summary rows for environmental hazards grid scoring.""" + settings = QSettings() + working_directory = settings.value("last_working_directory", "") + if not working_directory or not os.path.exists(working_directory): + QMessageBox.warning( + self, + "No Working Directory", + "No valid working directory found. Please create or open a project first.", + ) + return + + study_area_gpkg = os.path.join(working_directory, "study_area", "study_area.gpkg") + if not os.path.exists(study_area_gpkg): + QMessageBox.warning( + self, + "Study Area Required", + "Study area GeoPackage not found. Please create a project first.", + ) + return + + hazard_field = self._hazard_field_from_attributes() + if not hazard_field: + QMessageBox.warning(self, "S2S Field Required", "No S2S environmental hazards field is configured.") + return + + aoi_layer = self._build_aoi_layer(study_area_gpkg) + if not aoi_layer: + QMessageBox.warning( + self, + "Invalid Study Area", + "Could not load study_area_bboxes from study_area.gpkg.", + ) + return + + aoi_feature = S2SDataSourceWidget._build_aoi_feature(aoi_layer) + if not aoi_feature: + QMessageBox.warning(self, "Invalid AOI", "Failed to build AOI feature from study area geometry.") + return + + filename = f"s2s_environmental_hazards_{self.attributes.get('id', '').lower()}" + self.s2s_vector_output_path = os.path.join(working_directory, "study_area", f"{filename}.gpkg") + self.s2s_raster_output_path = "" + self.s2s_ntl_field = hazard_field + + gate_label = f"widget:hazard:{self.attributes.get('id', '').lower()}" + token = S2STaskGate.acquire(gate_label) + if not token: + active = S2STaskGate.active_label() or "another panel" + QMessageBox.information( + self, + "S2S Busy", + f"Another S2S download is currently running ({active}). Please wait for it to finish.", + ) + return + self._s2s_gate_token = token + + self.s2s_controls.set_running() + self._set_status("Fetching S2S data...") + self._s2s_error_handled = False + + self.s2s_task = S2SDownloaderTask( + aoi=aoi_feature, + fields=[hazard_field], + working_dir=working_directory, + filename=filename, + spatial_join_method="centroid", + geometry="point", + delete_existing=True, + ) + self.s2s_task.progress_updated.connect(self._on_s2s_progress) + self.s2s_task.error_occurred.connect(self._on_s2s_error) + self.s2s_task.taskCompleted.connect(self._on_s2s_completed) + self.s2s_task.taskTerminated.connect(self._on_s2s_terminated) + QgsApplication.taskManager().addTask(self.s2s_task) + + def _hazard_field_from_attributes(self) -> str: + """Resolve S2S hazard field from indicator id or existing attribute.""" + existing = self.attributes.get("s2s_hazard_field", "") + if existing: + return str(existing) + indicator_id = str(self.attributes.get("id", "")).lower() + return DEFAULT_S2S_ENV_HAZARD_FIELDS.get(indicator_id, "") + + def _resolve_default_s2s_output_path(self, working_directory: str) -> str: + """Return default hazard-specific S2S output path for this indicator.""" + if not working_directory: + return "" + indicator_id = str(self.attributes.get("id", "")).lower() + if not indicator_id: + return "" + filename = f"s2s_environmental_hazards_{indicator_id}.gpkg" + return os.path.join(working_directory, "study_area", filename) + + @staticmethod + def _build_aoi_layer(study_area_gpkg: str): + """Build and validate AOI layer from study area geopackage.""" + aoi_layer = QgsVectorLayer(f"{study_area_gpkg}|layername=study_area_bboxes", "study_area_bboxes", "ogr") + if not aoi_layer.isValid() or aoi_layer.featureCount() == 0: + return None + return aoi_layer + + def select_raster(self) -> None: + """Select raster or vector file for environmental hazards input.""" + last_dir = self.settings.value("GeoE3/lastRasterDir", "") + if not last_dir: + last_dir = self.settings.value("GeoE3/lastShapefileDir", "") + indicator_name = self.attributes.get("name") or "Environmental Hazards" + file_path, _ = QFileDialog.getOpenFileName( + self, + f"Select {indicator_name} Layer", + last_dir, + "Supported (*.vrt *.tif *.asc *.gpkg *.shp *.geojson *.json *.sqlite *.fgb *.parquet);;" + "Raster (*.vrt *.tif *.asc);;" + "Vector (*.gpkg *.shp *.geojson *.json *.sqlite *.fgb *.parquet);;" + "All files (*)", + ) + if not file_path: + return + + self.raster_layer_combo.setVisible(False) + self.raster_line_edit.setVisible(True) + self.raster_line_edit.setText(file_path) + parent_directory = os.path.dirname(file_path) + self.settings.setValue("GeoE3/lastRasterDir", parent_directory) + self.settings.setValue("GeoE3/lastShapefileDir", parent_directory) + self.resizeEvent(None) + self._update_vector_field_combo() + + def update_attributes(self): + """Update attributes with hazard-specific S2S metadata.""" + super().update_attributes() + self.attributes["s2s_hazard_field"] = self.s2s_ntl_field + self.attributes["s2s_ntl_field"] = "" + + def _select_existing_hazard_output_layer(self) -> None: + """Auto-select existing S2S hazard output when available.""" + if not self.s2s_vector_output_path: + self.s2s_vector_output_path = self.attributes.get("s2s_output_path", "") + if not self.s2s_vector_output_path or not os.path.exists(self.s2s_vector_output_path): + return + + layer_name = os.path.splitext(os.path.basename(self.s2s_vector_output_path))[0] + output_layer = S2SDataSourceWidget._load_or_reuse_vector_layer(self.s2s_vector_output_path, layer_name) + if output_layer is None: + self._set_status("S2S output invalid") + return + + self.raster_line_edit.clear() + self.raster_line_edit.setVisible(False) + self.raster_layer_combo.setVisible(True) + self.raster_layer_combo.setLayer(output_layer) + self.s2s_controls.set_downloaded() + self.update_attributes() diff --git a/geest/gui/widgets/datasource_widgets/s2s_ntl_raster_datasource_widget.py b/geest/gui/widgets/datasource_widgets/s2s_ntl_raster_datasource_widget.py new file mode 100644 index 00000000..ffbbd1f8 --- /dev/null +++ b/geest/gui/widgets/datasource_widgets/s2s_ntl_raster_datasource_widget.py @@ -0,0 +1,301 @@ +# -*- coding: utf-8 -*- +"""S2S-backed Nighttime Lights raster datasource widget.""" + +import os +from urllib.parse import quote + +from qgis.core import ( + QgsApplication, + QgsMapLayerType, + QgsMapLayerProxyModel, + QgsVectorLayer, +) +from qgis.PyQt.QtCore import QSettings +from qgis.PyQt.QtWidgets import QFileDialog, QLabel, QMessageBox, QSizePolicy + +from geest.core import S2STaskGate +from geest.core.constants import DEFAULT_S2S_NTL_FIELD +from geest.core.tasks import S2SDownloaderTask + +from .download_task_controls import DownloadTaskControls +from .raster_datasource_widget import RasterDataSourceWidget +from .s2s_datasource_widget import S2SDataSourceWidget + + +class S2SNTLRasterDataSourceWidget(RasterDataSourceWidget): + """Datasource widget that fetches NTL from S2S for regional grid scoring.""" + + VECTOR_EXTENSIONS = {".gpkg", ".shp", ".geojson", ".json", ".sqlite", ".fgb", ".parquet"} + + def add_internal_widgets(self) -> None: + """Build raster controls and append S2S fetch controls.""" + super().add_internal_widgets() + + self.raster_layer_combo.setFilters(QgsMapLayerProxyModel.RasterLayer | QgsMapLayerProxyModel.VectorLayer) + self.raster_layer_combo.setToolTip("Select raster or vector layer from the map") + self.raster_layer_combo.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed) + + existing_s2s_field = getattr(self, "s2s_ntl_field", "") + self.s2s_ntl_field = existing_s2s_field or self.attributes.get("s2s_ntl_field") or DEFAULT_S2S_NTL_FIELD + self.s2s_controls = DownloadTaskControls( + button_text="Download from S2S", + tooltip="Download data from Space2Stats", + click_handler=self.fetch_from_s2s, + ) + self.s2s_fetch_button = self.s2s_controls.button + self.layout.addWidget(self.s2s_controls.container) + + self.s2s_status_label = QLabel() + self.s2s_status_label.setSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Fixed) + self.s2s_status_label.setMinimumWidth(90) + self.s2s_status_label.setMaximumWidth(170) + self.layout.addWidget(self.s2s_status_label) + + self.raster_line_edit.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed) + self.layout.setStretchFactor(self.raster_layer_combo, 5) + self.layout.setStretchFactor(self.raster_line_edit, 5) + + self.s2s_task = None + self._s2s_gate_token = None + self.s2s_vector_output_path = self.attributes.get("s2s_output_path", "") + self.s2s_raster_output_path = "" + self._s2s_error_handled = False + + if not self.s2s_vector_output_path: + settings = QSettings() + working_directory = settings.value("last_working_directory", "") + candidate_path = self._resolve_default_s2s_output_path(working_directory) + if candidate_path and os.path.exists(candidate_path): + self.s2s_vector_output_path = candidate_path + if self.s2s_vector_output_path and os.path.exists(self.s2s_vector_output_path): + self._select_existing_s2s_output_layer() + + def _resolve_default_s2s_output_path(self, working_directory: str) -> str: + """Return default S2S output path for this widget.""" + if not working_directory: + return "" + return os.path.join(working_directory, "study_area", "s2s_nighttime_lights.gpkg") + + def fetch_from_s2s(self) -> None: + """Fetch S2S summary rows for downstream grid-based regional scoring.""" + settings = QSettings() + working_directory = settings.value("last_working_directory", "") + if not working_directory or not os.path.exists(working_directory): + QMessageBox.warning( + self, + "No Working Directory", + "No valid working directory found. Please create or open a project first.", + ) + return + + study_area_gpkg = os.path.join(working_directory, "study_area", "study_area.gpkg") + if not os.path.exists(study_area_gpkg): + QMessageBox.warning( + self, + "Study Area Required", + "Study area GeoPackage not found. Please create a project first.", + ) + return + + ntl_field = self.s2s_ntl_field + if not ntl_field: + QMessageBox.warning(self, "S2S Field Required", "No S2S nighttime lights field is configured.") + return + + aoi_layer = QgsVectorLayer(f"{study_area_gpkg}|layername=study_area_bboxes", "study_area_bboxes", "ogr") + if not aoi_layer.isValid() or aoi_layer.featureCount() == 0: + QMessageBox.warning( + self, + "Invalid Study Area", + "Could not load study_area_bboxes from study_area.gpkg.", + ) + return + + aoi_feature = S2SDataSourceWidget._build_aoi_feature(aoi_layer) + if not aoi_feature: + QMessageBox.warning(self, "Invalid AOI", "Failed to build AOI feature from study area geometry.") + return + + filename = "s2s_nighttime_lights" + self.s2s_vector_output_path = os.path.join(working_directory, "study_area", f"{filename}.gpkg") + self.s2s_raster_output_path = "" + + gate_label = "widget:nighttime_lights" + token = S2STaskGate.acquire(gate_label) + if not token: + active = S2STaskGate.active_label() or "another panel" + QMessageBox.information( + self, + "S2S Busy", + f"Another S2S download is currently running ({active}). Please wait for it to finish.", + ) + return + self._s2s_gate_token = token + + self.s2s_controls.set_running() + self._set_status("Fetching S2S data...") + self._s2s_error_handled = False + + self.s2s_task = S2SDownloaderTask( + aoi=aoi_feature, + fields=[ntl_field], + working_dir=working_directory, + filename=filename, + spatial_join_method="centroid", + geometry="point", + delete_existing=True, + ) + self.s2s_task.progress_updated.connect(self._on_s2s_progress) + self.s2s_task.error_occurred.connect(self._on_s2s_error) + self.s2s_task.taskCompleted.connect(self._on_s2s_completed) + self.s2s_task.taskTerminated.connect(self._on_s2s_terminated) + QgsApplication.taskManager().addTask(self.s2s_task) + + def _on_s2s_progress(self, message: str) -> None: + """Update S2S status text from task progress.""" + self._set_status(message) + self.s2s_controls.update_progress(message) + + def _on_s2s_error(self, message: str) -> None: + """Handle S2S task errors.""" + self._s2s_error_handled = True + self._release_s2s_gate() + self._set_status("S2S download failed") + self.s2s_controls.set_download_failed(message) + friendly_message = S2SDataSourceWidget._humanize_s2s_error(message) + QMessageBox.warning(self, "S2S Download Failed", friendly_message) + + def _on_s2s_terminated(self) -> None: + """Handle cancelled/terminated S2S tasks.""" + self._release_s2s_gate() + if self._s2s_error_handled: + return + self._set_status("S2S task terminated") + self.s2s_controls.set_cancelled() + + def _on_s2s_completed(self) -> None: + """Record S2S vector output and update attributes for grid-based workflows.""" + self._release_s2s_gate() + self.s2s_controls.reset() + + if not os.path.exists(self.s2s_vector_output_path): + self._set_status("S2S output not found") + self.s2s_controls.set_not_found(self.s2s_vector_output_path) + return + + layer_name = os.path.splitext(os.path.basename(self.s2s_vector_output_path))[0] + s2s_layer = S2SDataSourceWidget._load_or_reuse_vector_layer(self.s2s_vector_output_path, layer_name) + if s2s_layer is None: + self.s2s_controls.set_load_failed(self.s2s_vector_output_path) + self._set_status("S2S output invalid") + QMessageBox.warning(self, "Invalid S2S Output", "S2S output file exists but could not be loaded.") + return + + self.raster_line_edit.clear() + self.raster_line_edit.setVisible(False) + self.raster_layer_combo.setVisible(True) + self.raster_layer_combo.setLayer(s2s_layer if s2s_layer.isValid() else None) + + self._set_status("S2S nighttime lights downloaded") + self.s2s_controls.set_downloaded() + self.update_attributes() + + def _release_s2s_gate(self) -> None: + """Release global S2S gate lock for this widget task.""" + if getattr(self, "_s2s_gate_token", None): + S2STaskGate.release(self._s2s_gate_token) + self._s2s_gate_token = None + + def _set_status(self, message: str) -> None: + """Set status label text when available.""" + if hasattr(self, "s2s_status_label") and self.s2s_status_label is not None: + self.s2s_status_label.setText(message) + + def _select_existing_s2s_output_layer(self) -> None: + """Auto-select existing S2S nighttime lights layer from disk.""" + if not self.s2s_vector_output_path or not os.path.exists(self.s2s_vector_output_path): + return + + layer_name = os.path.splitext(os.path.basename(self.s2s_vector_output_path))[0] + output_layer = S2SDataSourceWidget._load_or_reuse_vector_layer(self.s2s_vector_output_path, layer_name) + if output_layer is None: + self._set_status("S2S output invalid") + return + + self.raster_line_edit.clear() + self.raster_line_edit.setVisible(False) + self.raster_layer_combo.setVisible(True) + self.raster_layer_combo.setLayer(output_layer) + self.s2s_controls.set_downloaded() + self.update_attributes() + + def select_raster(self) -> None: + """Select raster or vector file for nighttime lights input.""" + last_dir = self.settings.value("GeoE3/lastRasterDir", "") + if not last_dir: + last_dir = self.settings.value("GeoE3/lastShapefileDir", "") + file_path, _ = QFileDialog.getOpenFileName( + self, + "Select Nighttime Lights Layer", + last_dir, + "Supported (*.vrt *.tif *.asc *.gpkg *.shp *.geojson *.json *.sqlite *.fgb *.parquet);;" + "Raster (*.vrt *.tif *.asc);;" + "Vector (*.gpkg *.shp *.geojson *.json *.sqlite *.fgb *.parquet);;" + "All files (*)", + ) + if not file_path: + return + + self.raster_layer_combo.setVisible(False) + self.raster_line_edit.setVisible(True) + self.raster_line_edit.setText(file_path) + parent_directory = os.path.dirname(file_path) + self.settings.setValue("GeoE3/lastRasterDir", parent_directory) + self.settings.setValue("GeoE3/lastShapefileDir", parent_directory) + self.resizeEvent(None) + + def clear_raster(self): + """Clear selected file and reset widget state.""" + super().clear_raster() + + @classmethod + def _is_vector_path(cls, file_path: str) -> bool: + """Return True when file extension represents a known vector format.""" + if not file_path: + return False + extension = os.path.splitext(file_path)[1].lower() + return extension in cls.VECTOR_EXTENSIONS + + def update_attributes(self): + """Update raster attributes and S2S metadata.""" + super().update_attributes() + selected_layer = self.raster_layer_combo.currentLayer() + selected_path = self.raster_line_edit.text().strip() + is_vector_file = self._is_vector_path(selected_path) + is_vector_layer = bool(selected_layer and selected_layer.type() == QgsMapLayerType.VectorLayer) + + self.attributes[f"{self.widget_key}_input_type"] = "none" + + if is_vector_file: + self.attributes[f"{self.widget_key}_vector"] = quote(selected_path) + self.attributes[f"{self.widget_key}_raster"] = "" + self.attributes[f"{self.widget_key}_selected_field"] = "" + self.attributes[f"{self.widget_key}_input_type"] = "vector" + elif is_vector_layer and self._is_vector_path(selected_layer.source()): + self.attributes[f"{self.widget_key}_vector"] = quote(selected_layer.source()) + self.attributes[f"{self.widget_key}_selected_field"] = "" + self.attributes[f"{self.widget_key}_input_type"] = "vector" + elif selected_path: + self.attributes[f"{self.widget_key}_input_type"] = "raster" + else: + self.attributes[f"{self.widget_key}_vector"] = "" + self.attributes[f"{self.widget_key}_selected_field"] = "" + if selected_layer: + self.attributes[f"{self.widget_key}_input_type"] = "raster" + + if not hasattr(self, "s2s_ntl_field"): + return + self.attributes["s2s_ntl_field"] = self.s2s_ntl_field + self.attributes["s2s_spatial_join_method"] = "centroid" + if self.s2s_vector_output_path: + self.attributes["s2s_output_path"] = self.s2s_vector_output_path diff --git a/geest/gui/widgets/datasource_widgets/vector_and_field_datasource_widget.py b/geest/gui/widgets/datasource_widgets/vector_and_field_datasource_widget.py index 5ff81cdb..a7f5259e 100644 --- a/geest/gui/widgets/datasource_widgets/vector_and_field_datasource_widget.py +++ b/geest/gui/widgets/datasource_widgets/vector_and_field_datasource_widget.py @@ -156,14 +156,19 @@ def resize_clear_button(self): def select_shapefile(self): """ - Opens a file dialog to select a shapefile and stores the last directory in QSettings. + Opens a file dialog to select a vector file and stores the last directory in QSettings. """ try: settings = QSettings() last_dir = settings.value("GeoE3/lastShapefileDir", "") - # Open file dialog to select a shapefile - file_path, _ = QFileDialog.getOpenFileName(self, "Select Shapefile", last_dir, "Shapefiles (*.shp)") + # Open file dialog to select a vector file + file_path, _ = QFileDialog.getOpenFileName( + self, + "Select Vector File", + last_dir, + "GeoPackage and Shapefiles (*.gpkg *.shp);;GeoPackage (*.gpkg);;Shapefiles (*.shp)", + ) if file_path: # Update the line edit with the selected file path @@ -222,43 +227,86 @@ def update_selected_field(self) -> None: """ Updates the selected field in the attributes dictionary when the field selection changes. """ - if not self.field_selection_combo.isEnabled(): - self.attributes[f"{self.widget_key}_selected_field"] = None + try: + if not self.field_selection_combo.isEnabled(): + self.attributes[f"{self.widget_key}_selected_field"] = None + self.data_changed.emit(self.attributes) + return + + selected_field = (self.field_selection_combo.currentText() or "").strip() + self.attributes[f"{self.widget_key}_selected_field"] = selected_field + if not selected_field: + self.data_changed.emit(self.attributes) + return + + # Store the selected field in QSettings + self.settings.setValue(f"{self.widget_key}_selected_field", selected_field) + if self._supports_unique_value_mapping(): + vector_layer = self._resolve_layer_for_unique_values() + values_dict = {} + + if vector_layer is not None: + fields = vector_layer.fields() + if fields is not None: + idx = fields.indexOf(selected_field) + if idx >= 0: + values = vector_layer.uniqueValues(idx) + # Keep string keys only to avoid QVariant/empty-row artifacts. + for value in values: + if isinstance(value, str): + values_dict[value] = None + + # Preserve existing values if they exist + existing_values = self.attributes.get(f"{self.widget_key}_unique_values", {}) + + for key in values_dict.keys(): + if key not in existing_values: + values_dict[key] = None + else: + values_dict[key] = existing_values[key] + log_message(f"Existing values: {existing_values}") + log_message(f"New values: {values_dict}") + # will drop any keys in the json item that are not in values_dict + self.attributes[f"{self.widget_key}_unique_values"] = values_dict self.data_changed.emit(self.attributes) + except (RuntimeError, AttributeError): + # Can occur while dialog widgets/layers are being destroyed during close. return - selected_field = self.field_selection_combo.currentText() - self.attributes[f"{self.widget_key}_selected_field"] = selected_field + def _supports_unique_value_mapping(self) -> bool: + """Return True when this widget should maintain unique value mappings.""" + return ( + self.attributes.get("id", None) == "Street_Lights" + or bool(self.attributes.get("use_classify_polygon_into_classes", 0)) + or bool(self.attributes.get("use_classify_safety_polygon_into_classes", 0)) + ) + + def _resolve_layer_for_unique_values(self): + """Resolve the layer used to derive unique values. - # Store the selected field in QSettings - self.settings.setValue(f"{self.widget_key}_selected_field", selected_field) - if self.attributes.get("id", None) == "Street_Lights": - # retrieve the unique values for the selected field + Priority: + 1. Active layer selected in combo box. + 2. Vector file path set in shapefile line edit. + """ + try: vector_layer = self.layer_combo.currentLayer() - idx = vector_layer.fields().indexOf(selected_field) - values = vector_layer.uniqueValues(idx) - values_dict = {} - - # list the data type of each value - for value in values: - # log_message(f"{type(value)} value {value}") - # Dont remove this! It cleans to contents to remove QVariants - # introduced from empty table rows! - if isinstance(value, str): - values_dict[value] = None - # Preserve existing values if they exist - existing_values = self.attributes.get(f"{self.widget_key}_unique_values", {}) - - for key in values_dict.keys(): - if key not in existing_values: - values_dict[key] = None - else: - values_dict[key] = existing_values[key] - log_message(f"Existing values: {existing_values}") - log_message(f"New values: {values_dict}") - # will drop any keys in the json item that are not in values_dict - self.attributes[f"{self.widget_key}_unique_values"] = values_dict - self.data_changed.emit(self.attributes) + except RuntimeError: + return None + if vector_layer: + return vector_layer + + try: + shapefile_path = unquote(self.shapefile_line_edit.text()).strip() + except RuntimeError: + return None + if not shapefile_path: + return None + + vector_layer = QgsVectorLayer(shapefile_path, "layer", "ogr") + if not vector_layer.isValid(): + log_message(f"Failed to load vector file for unique value extraction: {shapefile_path}", level=Qgis.Warning) + return None + return vector_layer def update_field_combo(self) -> None: """ diff --git a/geest/gui/widgets/datasource_widgets/vector_datasource_widget.py b/geest/gui/widgets/datasource_widgets/vector_datasource_widget.py index f9b1df01..36e128e9 100644 --- a/geest/gui/widgets/datasource_widgets/vector_datasource_widget.py +++ b/geest/gui/widgets/datasource_widgets/vector_datasource_widget.py @@ -18,17 +18,14 @@ QgsVectorLayer, ) from qgis.gui import QgsMapLayerComboBox -from qgis.PyQt.QtCore import QSettings, Qt, QTimer -from qgis.PyQt.QtGui import QFont, QIcon, QMovie +from qgis.PyQt.QtCore import QSettings, Qt +from qgis.PyQt.QtGui import QFont, QIcon from qgis.PyQt.QtWidgets import ( QFileDialog, - QHBoxLayout, QLabel, QLineEdit, QMessageBox, - QPushButton, QToolButton, - QWidget, ) from geest.core.osm_downloaders import OSMDownloadType @@ -36,6 +33,7 @@ from geest.utilities import log_message, resources_path from .base_datasource_widget import BaseDataSourceWidget +from .download_task_controls import DownloadTaskControls class VectorDataSourceWidget(BaseDataSourceWidget): @@ -290,31 +288,17 @@ def add_internal_widgets(self) -> None: self.osm_spinner_label = None self.osm_spinner_movie = None self.osm_button_container = None + self.osm_controls = None if self.should_add_osm_widget: - # Create container widget for button and spinner - self.osm_button_container = QWidget() - container_layout = QHBoxLayout(self.osm_button_container) - container_layout.setContentsMargins(0, 0, 0, 0) - container_layout.setSpacing(6) - - self.osm_download_button = QPushButton(self.osm_button_text) - self.osm_download_button.setToolTip(self.osm_tooltip) - self.osm_download_button.setStyleSheet("padding: 5px 10px;") - self.osm_download_button.clicked.connect(self.start_osm_download) - - # Create spinner label with animated gif - self.osm_spinner_label = QLabel() - self.osm_spinner_movie = QMovie(resources_path("resources", "throbber.gif")) - # Scale the spinner to match button height - self.osm_spinner_movie.setScaledSize( - self.osm_spinner_movie.currentPixmap().size().scaled(24, 24, Qt.KeepAspectRatio) + self.osm_controls = DownloadTaskControls( + button_text=self.osm_button_text, + tooltip=self.osm_tooltip, + click_handler=self.start_osm_download, ) - self.osm_spinner_label.setMovie(self.osm_spinner_movie) - self.osm_spinner_label.setVisible(False) # Hidden initially - - container_layout.addWidget(self.osm_download_button) - container_layout.addWidget(self.osm_spinner_label) - container_layout.addStretch() + self.osm_button_container = self.osm_controls.container + self.osm_download_button = self.osm_controls.button + self.osm_spinner_label = self.osm_controls.spinner_label + self.osm_spinner_movie = self.osm_controls.spinner_movie log_message( f"OSM download button created for indicator: {self.attributes.get('id', 'unknown')}", @@ -379,14 +363,19 @@ def resize_clear_button(self): def select_shapefile(self): """ - Opens a file dialog to select a shapefile and stores the last directory in QSettings. + Opens a file dialog to select a vector file and stores the last directory in QSettings. """ try: settings = QSettings() last_dir = settings.value("GeoE3/lastShapefileDir", "") - # Open file dialog to select a shapefile - file_path, _ = QFileDialog.getOpenFileName(self, "Select Shapefile", last_dir, "Shapefiles (*.shp)") + # Open file dialog to select a vector file + file_path, _ = QFileDialog.getOpenFileName( + self, + "Select Vector File", + last_dir, + "GeoPackage and Shapefiles (*.gpkg *.shp);;GeoPackage (*.gpkg);;Shapefiles (*.shp)", + ) if file_path: # Update the line edit with the selected file path @@ -515,13 +504,7 @@ def start_osm_download(self) -> None: log_message(f"Output CRS: {output_crs.authid()}", tag="GeoE3", level=Qgis.Info) if self.osm_download_button: - self.osm_download_button.setEnabled(False) - self.osm_download_button.setText("Downloading...") - self.osm_download_button.setStyleSheet("padding: 5px 10px;") - # Start the spinner animation - if self.osm_spinner_label and self.osm_spinner_movie: - self.osm_spinner_label.setVisible(True) - self.osm_spinner_movie.start() + self.osm_controls.set_running() try: # Create task using proper QgsTask-based approach @@ -554,10 +537,7 @@ def start_osm_download(self) -> None: log_message(traceback.format_exc(), tag="GeoE3", level=Qgis.Critical) if self.osm_download_button: - self.osm_download_button.setEnabled(True) - self.osm_download_button.setText("Download from OSM") - # Stop the spinner on error - self._stop_spinner() + self.osm_controls.set_error(str(e)) QMessageBox.warning(self, "Error", f"Failed to start download: {str(e)}") @@ -569,10 +549,7 @@ def update_button_progress(self, message: str): """ log_message(message, tag="GeoE3", level=Qgis.Info) if self.osm_download_button: - if "Processing" in message: - self.osm_download_button.setText("Processing...") - elif "complete" in message.lower(): - self.osm_download_button.setText("Complete!") + self.osm_controls.update_progress(message) def on_osm_download_finished(self, gpkg_path: str) -> None: """Handle completion of OSM download. @@ -587,10 +564,7 @@ def on_osm_download_finished(self, gpkg_path: str) -> None: error_msg = f"Expected a file but received a directory: {gpkg_path}" log_message(f"Error: {error_msg}", tag="GeoE3", level=Qgis.Critical) if self.osm_download_button: - self.osm_download_button.setText("Error!") - self.osm_download_button.setStyleSheet("background-color: #ffcccc; padding: 5px 10px;") - self.osm_download_button.setEnabled(True) - self.osm_download_button.setToolTip(f"Error: {error_msg}") + self.osm_controls.set_error(error_msg) QMessageBox.warning(self, "OSM Download Error", error_msg) return @@ -598,10 +572,7 @@ def on_osm_download_finished(self, gpkg_path: str) -> None: error_msg = f"Download completed but output file not found: {gpkg_path}" log_message(f"Error: {error_msg}", tag="GeoE3", level=Qgis.Critical) if self.osm_download_button: - self.osm_download_button.setText("Not Found!") - self.osm_download_button.setStyleSheet("background-color: #ffcccc; padding: 5px 10px;") - self.osm_download_button.setEnabled(True) - self.osm_download_button.setToolTip(f"Error: {error_msg}") + self.osm_controls.set_not_found(gpkg_path) QMessageBox.warning(self, "OSM Download Error", error_msg) return @@ -614,17 +585,12 @@ def on_osm_download_finished(self, gpkg_path: str) -> None: self.layer_combo.setLayer(layer) if self.osm_download_button: - self.osm_download_button.setText("Downloaded!") - self.osm_download_button.setStyleSheet("background-color: #ccffcc; padding: 5px 10px;") - QTimer.singleShot(2000, lambda: self.reset_osm_button()) + self.osm_controls.set_downloaded() else: error_msg = f"Downloaded file exists but could not be loaded as a valid layer: {gpkg_path}" log_message(f"Failed to load layer: {error_msg}", tag="GeoE3", level=Qgis.Critical) if self.osm_download_button: - self.osm_download_button.setText("Load Failed!") - self.osm_download_button.setStyleSheet("background-color: #ffcccc; padding: 5px 10px;") - self.osm_download_button.setEnabled(True) - self.osm_download_button.setToolTip(f"Error: {error_msg}") + self.osm_controls.set_load_failed(gpkg_path) QMessageBox.warning( self, "OSM Layer Load Failed", @@ -646,11 +612,7 @@ def on_osm_download_error(self, error_message: str) -> None: self._stop_spinner() if self.osm_download_button: - self.osm_download_button.setText("Download Failed!") - self.osm_download_button.setStyleSheet("background-color: #ffcccc; padding: 5px 10px;") - self.osm_download_button.setEnabled(True) - # Set tooltip with error details so user can hover to see what went wrong - self.osm_download_button.setToolTip(f"Error: {error_message}\n\nClick to retry.") + self.osm_controls.set_download_failed(error_message) # Show error message to user QMessageBox.warning( @@ -674,26 +636,17 @@ def on_osm_download_terminated(self) -> None: self._stop_spinner() if self.osm_download_button: - self.osm_download_button.setText("Cancelled") - self.osm_download_button.setStyleSheet("background-color: #ffffcc; padding: 5px 10px;") - self.osm_download_button.setEnabled(True) - self.osm_download_button.setToolTip("Download was cancelled. Click to retry.") + self.osm_controls.set_cancelled() def _stop_spinner(self) -> None: """Stop the spinner animation and hide it.""" - if self.osm_spinner_movie: - self.osm_spinner_movie.stop() - if self.osm_spinner_label: - self.osm_spinner_label.setVisible(False) + if self.osm_controls: + self.osm_controls.stop_spinner() def reset_osm_button(self) -> None: """Reset OSM download button to initial state.""" - if self.osm_download_button: - self.osm_download_button.setText("Download from OSM") - self.osm_download_button.setStyleSheet("padding: 5px 10px;") - self.osm_download_button.setEnabled(True) - self.osm_download_button.setToolTip(self.osm_tooltip) # Restore original tooltip - self._stop_spinner() + if self.osm_controls: + self.osm_controls.reset() def get_osm_download_button(self): """ diff --git a/geest/resources/model.json b/geest/resources/model.json index db0b563c..b61dec11 100644 --- a/geest/resources/model.json +++ b/geest/resources/model.json @@ -133,8 +133,8 @@ "dimension_weighting": 0.333333, "indicators": [ { - "indicator": "WBL 2024 Entrepeneurship Index Score", - "id": "Entrepeneurship_Index", + "indicator": "WBL 2024 Entrepreneurship Index Score", + "id": "Entrepreneurship_Index", "output_filename": "FIN_output", "description": "", "default_factor_weighting": 1.0, @@ -333,7 +333,7 @@ "indicators": [ { "indicator": "Location of public transportation stops, including maritime", - "id": "Pulic_Transport_location", + "id": "Public_Transport_location", "output_filename": "PBT_output", "description": "", "default_factor_weighting": 1.0, @@ -608,7 +608,7 @@ "id": "education", "output_filename": "education", "name": "Education", - "women_enabling": 0, + "women_enabling": 1, "default_dimension_weighting": 0.142857142857143, "dimension_weighting": 0.142857142857143, "indicators": [ @@ -631,7 +631,16 @@ "use_classify_polygon_into_classes": 1, "use_classify_safety_polygon_into_classes": 0, "use_csv_to_point_layer": 0, - "use_polygon_per_cell": 0, + "use_polygon_per_cell": 1, + "s2s_fields": [ + "ghs_11_pop", + "ghs_12_pop", + "ghs_13_pop", + "ghs_22_pop", + "ghs_23_pop", + "ghs_30_pop", + "ghs_total_pop" + ], "use_polyline_per_cell": 0, "use_point_per_cell": 0, "use_nighttime_lights": 0, @@ -868,4 +877,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/geest/resources/qml/indicator-vector-template.qml b/geest/resources/qml/indicator-vector-template.qml new file mode 100644 index 00000000..8f8316c5 --- /dev/null +++ b/geest/resources/qml/indicator-vector-template.qml @@ -0,0 +1,269 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + 0 + 2 + diff --git a/geest/resources/qpt/analysis_summary_report_template.qpt b/geest/resources/qpt/analysis_summary_report_template.qpt index 36a0181e..3e9ab6a4 100644 --- a/geest/resources/qpt/analysis_summary_report_template.qpt +++ b/geest/resources/qpt/analysis_summary_report_template.qpt @@ -1,80 +1,80 @@ - - - + + + - + - + - - - + + + - + - + @@ -82,225 +82,140 @@ - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - + - + - - - + + + - + - - - - + + + + - + - + - - - + + + @@ -308,89 +223,73 @@ - - - + + + - - - - - - - - - - - - - - - - - - - + + + + + + - - + + + + - - - + + - + + + + + + + + + + + - + diff --git a/geest/resources/qpt/study_area_report_template.qpt b/geest/resources/qpt/study_area_report_template.qpt index 2a2334a7..90b25fab 100644 --- a/geest/resources/qpt/study_area_report_template.qpt +++ b/geest/resources/qpt/study_area_report_template.qpt @@ -1,80 +1,80 @@ - - - + + + - + - + - - - + + + - + - + @@ -82,240 +82,240 @@ - - - + + + - + - - - - + + + + - + - + - - - + + + - + - - - - + + + + - + + + + + + + + + + + + - + - - - + + + - + - - - - + + + + - + - - - - - - - - - - - - + - - - + + + @@ -323,291 +323,198 @@ - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - - - - - - - - - - - - - - - - - - + + + + + + - - + + + + - - - + + + - + - - - - + + + + - + - + - - - + + + - - - - - - - - - - - - - - - - - - - + + + + + + - - + + + + - - - + + - - - - + + + - + + + + + + + + + + + - + diff --git a/geest/ui/create_project_panel_base.ui b/geest/ui/create_project_panel_base.ui index 55ef7373..bcf9eed6 100644 --- a/geest/ui/create_project_panel_base.ui +++ b/geest/ui/create_project_panel_base.ui @@ -361,8 +361,8 @@ When this option is **not selected**, the analysis focuses on generic employment - - + + Qt::Vertical @@ -374,8 +374,8 @@ When this option is **not selected**, the analysis focuses on generic employment - - + + 0 @@ -561,9 +561,9 @@ When this option is **not selected**, the analysis focuses on generic employment cell_size_spinbox layer_combo field_combo - load_boundary_button - use_boundary_crs - previous_button + load_boundary_button + use_boundary_crs + previous_button next_button diff --git a/geest/ui/credits_panel_base.ui b/geest/ui/credits_panel_base.ui index 1e6da320..ae3c00a5 100644 --- a/geest/ui/credits_panel_base.ui +++ b/geest/ui/credits_panel_base.ui @@ -197,7 +197,7 @@ - This plugin was built with support from the Canada Clean Energy and Forest Climate Facility (CCEFCF) and the Global Development Fund (GDF), by the Geospatial Team in the Development Economics Data Group (DECDG). + This plugin was built with support from the Canada Clean Energy and Forest Climate Facility (CCEFCF) and the Global Data Facility (GDF), by the Geospatial Team in the Development Economics Data Group (DECDG). The project is open-source and you can access the code here: https://github.com/worldbank/GEOE3 diff --git a/geest/ui/geoe3_settings_base.ui b/geest/ui/geoe3_settings_base.ui index 3ff4d6d1..2729174b 100644 --- a/geest/ui/geoe3_settings_base.ui +++ b/geest/ui/geoe3_settings_base.ui @@ -95,16 +95,26 @@ - - - - Filter study areas by GHSL settlements (skip areas without settlements) - - - true - - - + + + + Filter study areas by GHSL settlements (skip areas without settlements) + + + true + + + + + + + Source: GHSL (Global Human Settlement Layer), European Commission JRC. + + + true + + + @@ -175,16 +185,26 @@ - - - - Note: First run downloads large files; later runs reuse the cache. - - - true - - - + + + + Note: First run downloads large files; later runs reuse the cache. + + + true + + + + + + + Source: Ookla Open Data (Speedtest Global Performance), subject to provider terms. + + + true + + + diff --git a/geest/ui/ors_panel_base.ui b/geest/ui/ors_panel_base.ui index 9dc86ba6..31bffa27 100644 --- a/geest/ui/ors_panel_base.ui +++ b/geest/ui/ors_panel_base.ui @@ -257,11 +257,13 @@ 0 - - This plugin makes use of the Open Route Service (ORS) platform for elements of the spatial analysis workflows. In order to use ORS, you need to obtain an API key. There is no charge to get your key. Click on [this link](https://openrouteservice.org/dev/#/signup) for the API Key sign up page. Once you have your API key, paste it into the box below. + + This plugin makes use of the Open Route Service (ORS) platform for elements of the spatial analysis workflows. In order to use ORS, you need to obtain an API key. There is no charge to get your key. Click on [this link](https://openrouteservice.org/dev/#/signup) for the API Key sign up page. Once you have your API key, paste it into the box below. -Please note that you can write to ORS for a collaborator key if you experience issues with ORS timeouts during the analysis (there is no charge for this). - +Please note that you can write to ORS for a collaborator key if you experience issues with ORS timeouts during the analysis (there is no charge for this). + +Attribution: OpenRouteService by HeiGIT (https://openrouteservice.org). + Qt::MarkdownText diff --git a/geest/ui/s2s_panel_base.ui b/geest/ui/s2s_panel_base.ui new file mode 100644 index 00000000..346c1c48 --- /dev/null +++ b/geest/ui/s2s_panel_base.ui @@ -0,0 +1,223 @@ + + + SetupPanelBase + + + + 0 + 0 + 620 + 900 + + + + Form + + + + + + + + + ../resources/geoe3-banner.png + + + true + + + + + + + <html><head/><body><p align="center"><span style=" font-size:16pt; font-weight:600;">GeoE3 Space2Stats</span></p></body></html> + + + Qt::RichText + + + Qt::AlignCenter + + + + + + + Qt::Horizontal + + + + + + + QFrame::NoFrame + + + true + + + + + 0 + 0 + 602 + 646 + + + + + + + Optionally fetch Space2Stats datasets now for regional indicators. Fetching now reduces manual setup later in the indicator configuration tree. + +Attribution: Space2Stats API and datasets (https://api.space2stats.com). + + + Qt::PlainText + + + Qt::AlignJustify|Qt::AlignTop + + + true + + + + + + + Space2Stats + + + + + + Fetch S2S data + + + false + + + + + + + If enabled, GeoE3 will download Space2Stats datasets for all regional S2S-backed indicators when you click Next. + + + Qt::PlainText + + + Qt::AlignJustify|Qt::AlignTop + + + true + + + + + + + + + + false + + + + + + Qt::PlainText + + + Qt::AlignJustify|Qt::AlignTop + + + true + + + + + + + 0 + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + + + + + + + + 80 + 40 + + + + + 80 + 40 + + + + ◀️ + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + 80 + 40 + + + + + 80 + 40 + + + + ▶️ + + + + + + + + + + diff --git a/geest/utilities.py b/geest/utilities.py index 3c59bd7a..3a72c19f 100644 --- a/geest/utilities.py +++ b/geest/utilities.py @@ -69,7 +69,17 @@ def theme_stylesheet() -> str: # try move it to the top and check that all the subsequent rules work still... light_theme_stylesheet = f""" QPushButton {{ - background-color: rgba(62, 121, 155, 25); + background-color: rgba(62, 121, 155, 180); + color: #ffffff; + border: 1px solid #3E799B; + border-radius: 3px; + padding: 4px 8px; + }} + QPushButton:hover {{ + background-color: rgba(62, 121, 155, 220); + }} + QPushButton:pressed {{ + background-color: rgba(45, 90, 117, 255); }} QToolTip {{ color: #000000; diff --git a/scripts/start_qgis.sh b/scripts/start_qgis.sh index 4797b56f..2e224e7f 100755 --- a/scripts/start_qgis.sh +++ b/scripts/start_qgis.sh @@ -4,8 +4,8 @@ echo "--------------------------------" echo "Do you want to enable debug mode?" choice=$(gum choose "🪲 Yes" "🐞 No") case $choice in -"🪲 Yes") developer_mode=1 ;; -"🐞 No") developer_mode=0 ;; +"🪲 Yes") DEVELOPER_MODE=1 ;; +"🐞 No") DEVELOPER_MODE=0 ;; esac echo "Do you want to enable experimental features?" choice=$(gum choose "🪲 Yes" "🐞 No") @@ -21,14 +21,14 @@ rm -f "$GEOE3_LOG" #nix-shell -p \ # This is the old way using default nix packages with overrides # 'qgis.override { extraPythonPackages = (ps: [ ps.pyqtwebengine ps.jsonschema ps.debugpy ps.future ps.psutil ]);}' \ -# --command "GEEST_LOG=${GEEST_LOG} GEEST_DEBUG=${developer_mode} RUNNING_ON_LOCAL=1 qgis --profile GEEST2" +# --command "GEEST_LOG=${GEEST_LOG} GEEST_DEBUG=${DEVELOPER_MODE} RUNNING_ON_LOCAL=1 qgis --profile GEEST2" # This is the new way, using Ivan Mincis nix spatial project and a flake # see flake.nix for implementation details # Both GEOE3_* and GEEST_* env vars are set for backward compatibility # QT_QPA_PLATFORM flag forces it to run under x11 protocol GEOE3_LOG=${GEOE3_LOG} \ - GEOE3_DEBUG=${developer_mode} \ + GEOE3_DEBUG=${DEVELOPER_MODE} \ GEOE3_EXPERIMENTAL=${GEOE3_EXPERIMENTAL} \ GEOE3_TEST_DIR=${GEOE3_TEST_DIR} \ RUNNING_ON_LOCAL=1 \ diff --git a/scripts/start_qgis_ltr.sh b/scripts/start_qgis_ltr.sh index 5fa4a27b..2251a2e9 100755 --- a/scripts/start_qgis_ltr.sh +++ b/scripts/start_qgis_ltr.sh @@ -1,18 +1,36 @@ #!/usr/bin/env bash echo "🪛 Running QGIS with the GEOE3 profile:" echo "--------------------------------" +echo "Do you want to enable debug mode?" +choice=$(gum choose "🪲 Yes" "🐞 No") +case $choice in +"🪲 Yes") DEVELOPER_MODE=1 ;; +"🐞 No") DEVELOPER_MODE=0 ;; +esac +echo "Do you want to enable experimental features?" +choice=$(gum choose "🪲 Yes" "🐞 No") +case $choice in +"🪲 Yes") GEOE3_EXPERIMENTAL=1 ;; +"🐞 No") GEOE3_EXPERIMENTAL=0 ;; +esac -# Set environment variables (both GEOE3_* and GEEST_* for backward compatibility) +# Running on local used to skip tests that will not work in a local dev env +GEOE3_LOG=$HOME/GEOE3.log GEOE3_TEST_DIR="$(pwd)/test" -GEEST_TEST_DIR="$(pwd)/test" # Set test directory relative to project root +rm -f "$GEOE3_LOG" +#nix-shell -p \ +# This is the old way using default nix packages with overrides +# 'qgis.override { extraPythonPackages = (ps: [ ps.pyqtwebengine ps.jsonschema ps.debugpy ps.future ps.psutil ]);}' \ +# --command "GEEST_LOG=${GEEST_LOG} GEEST_DEBUG=${DEVELOPER_MODE} RUNNING_ON_LOCAL=1 qgis --profile GEEST2" -# This is the flake approach, using Ivan Mincis nix spatial project and a flake +# This is the new way, using Ivan Mincis nix spatial project and a flake # see flake.nix for implementation details +# Both GEOE3_* and GEEST_* env vars are set for backward compatibility # QT_QPA_PLATFORM flag forces it to run under x11 protocol GEOE3_LOG=${GEOE3_LOG} \ - GEEST_LOG=${GEEST_LOG} \ + GEOE3_DEBUG=${DEVELOPER_MODE} \ + GEOE3_EXPERIMENTAL=${GEOE3_EXPERIMENTAL} \ GEOE3_TEST_DIR=${GEOE3_TEST_DIR} \ - GEEST_TEST_DIR=${GEEST_TEST_DIR} \ RUNNING_ON_LOCAL=1 \ QT_QPA_PLATFORM=xcb \ nix run .#qgis-ltr -- --profile GEOE3