diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98218f42..3fb87ce7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: - name: Create test env shell: bash -l {0} run: | - conda create -n test_env python=3.12 libgdal=3.13 -c conda-forge -c defaults -y + conda create -n test_env python=3.12 pip libgdal=3.13 -c conda-forge -c defaults -y conda activate test_env PIP_NO_BINARY=rasterio pip install . - name: test diff --git a/setup.py b/setup.py index 2e0a2fd2..326fb599 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ 'pyscaffold', 'gdal>=3.5.0,<3.14', 'tqdm>=4.66', - 'numpy<2' + 'numpy' ], entry_points=""" [rasterio.rio_plugins] diff --git a/src/eolab/georastertools/processing/algo.py b/src/eolab/georastertools/processing/algo.py index 052b9c29..fe4e31c3 100644 --- a/src/eolab/georastertools/processing/algo.py +++ b/src/eolab/georastertools/processing/algo.py @@ -797,6 +797,6 @@ def hillshade(input_data : np.ndarray, elevation : float = 0.0, azimuth : float ratios = np.maximum(ratios, new_ratios) angles = np.arctan(ratios) - out[0, radius: shape[1] - radius, radius: shape[2] - radius] = angles > np.radians(elevation) + out[0, radius: shape[1] - radius, radius: shape[2] - radius] = angles > np.radians(elevation, dtype=np.float32) return out \ No newline at end of file diff --git a/src/eolab/georastertools/processing/stats.py b/src/eolab/georastertools/processing/stats.py index 5b53da99..c763cbed 100644 --- a/src/eolab/georastertools/processing/stats.py +++ b/src/eolab/georastertools/processing/stats.py @@ -479,18 +479,20 @@ def _gen_stats(dataset, stats: List[str] = None, feature_stats = dict() # compute stats + # we force dtype to float64 to avoid precision errors functions = { - 'min': np.ma.min, - 'max': np.ma.max, - 'mean': np.ma.mean, - 'sum': np.ma.sum, - 'std': np.ma.std, - 'median': np.ma.median + 'min': {'function': np.ma.min, 'options': {}}, + 'max': {'function': np.ma.max, 'options': {}}, + 'mean': {'function': np.ma.mean, 'options': {"dtype": np.float64}}, + 'sum': {'function': np.ma.sum, 'options': {"dtype": np.float64}}, + 'std': {'function': np.ma.std, 'options': {"dtype": np.float64}}, + 'median': {'function': np.ma.median, 'options': {}}, } - for key, function in functions.items(): + for key, key_content in functions.items(): if key in stats: - feature_stats[f'{prefix_stats}{key}'] = float(function(dataset)) + function = key_content["function"] + feature_stats[f'{prefix_stats}{key}'] = float(function(dataset, **key_content["options"])) if 'range' in stats: min_key = f'{prefix_stats}min' @@ -503,7 +505,7 @@ def _gen_stats(dataset, stats: List[str] = None, # because np.ma has no percentile computation capabilities dataset_com = dataset.compressed() for pctile in [s for s in stats if s.startswith('percentile_')]: - q = float(pctile.replace("percentile_", '')) + q = np.float64(pctile.replace("percentile_", '')) feature_stats[f'{prefix_stats}{pctile}'] = np.percentile(dataset_com, q) if 'mad' in stats: feature_stats[f'{prefix_stats}mad'] = median_abs_deviation(dataset_com.flatten()) diff --git a/tests/test_stats.py b/tests/test_stats.py index f7b5fe51..310164e8 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -50,7 +50,7 @@ def test_compute_zonal_default_stats(): [{'count': 64186, 'min': -1.0, 'max': 1.0, 'mean': 0.698426, 'std': 0.210732}], [{'count': 83063, 'min': -1.0, 'max': 1.0, 'mean': 0.699223, 'std': 0.213408}], [{'count': 4038, 'min': -0.206738, 'max': 1.0, 'mean': 0.734667, 'std': 0.230044}], - [{'count': 29232, 'min': -0.83908, 'max': 1.0, 'mean': 0.602069, 'std': 0.217282}], + [{'count': 29232, 'min': -0.83908, 'max': 1.0, 'mean': 0.60207, 'std': 0.217282}], [{'count': 177106, 'min': -1.0, 'max': 1.0, 'mean': 0.548052, 'std': 0.261178}], [{'count': 17772, 'min': 0.038081, 'max': 1.0, 'mean': 0.61795, 'std': 0.269253}], [{'count': 169829, 'min': -1.0, 'max': 1.0, 'mean': 0.525915, 'std': 0.258398}], @@ -85,79 +85,79 @@ def test_compute_zonal_extra_stats(): for geom_stats in statistics for d in geom_stats for key, val in d.items()] # ref is the following - ref = [[{'sum': 195899.96875, 'median': 0.636118, 'range': 2.0, + ref = [[{'sum': 195899.928989, 'median': 0.636118, 'range': 2.0, 'percentile_5': 0.15373, 'mad': 0.218207, 'majority': -1.0, 'minority': -0.996727, 'unique': 285995, 'nodata': 0, 'valid': 1.0}], - [{'sum': 31886.951172, 'median': 0.701654, 'range': 2.0, + [{'sum': 31886.95005, 'median': 0.701654, 'range': 2.0, 'percentile_5': 0.280309, 'mad': 0.161672, 'majority': 1.0, 'minority': -0.979167, 'unique': 44398, 'nodata': 367, 'valid': 0.992054}], - [{'sum': 7843.094727, 'median': 0.682968, 'range': 2.0, + [{'sum': 7843.094361, 'median': 0.682968, 'range': 2.0, 'percentile_5': 0.12968, 'mad': 0.146711, 'majority': -1.0, 'minority': -0.993174, 'unique': 12189, 'nodata': 0, 'valid': 1.0}], - [{'sum': 56049.859375, 'median': 0.668857, 'range': 2.0, + [{'sum': 56049.860487, 'median': 0.668857, 'range': 2.0, 'percentile_5': 0.154594, 'mad': 0.148523, 'majority': 1.0, 'minority': -0.977465, 'unique': 83944, 'nodata': 245, 'valid': 0.997214}], - [{'sum': 9305.114258, 'median': 0.713093, 'range': 0.911131, + [{'sum': 9305.114276, 'median': 0.713093, 'range': 0.911131, 'percentile_5': 0.358831, 'mad': 0.143713, 'majority': 0.6, 'minority': 0.088869, 'unique': 13137, 'nodata': 244, 'valid': 0.981971}], - [{'sum': 30402.613281, 'median': 0.753216, 'range': 1.656766, + [{'sum': 30402.614902, 'median': 0.753216, 'range': 1.656766, 'percentile_5': 0.433678, 'mad': 0.124387, 'majority': 1.0, 'minority': -0.656766, 'unique': 40146, 'nodata': 85, 'valid': 0.997948}], - [{'sum': 31683.017578, 'median': 0.471969, 'range': 1.927374, + [{'sum': 31683.020613, 'median': 0.471969, 'range': 1.927374, 'percentile_5': 0.110149, 'mad': 0.168122, 'majority': 0.5, 'minority': -0.927374, 'unique': 63745, 'nodata': 0, 'valid': 1.0}], - [{'sum': 56840.1875, 'median': 0.629144, 'range': 2.0, + [{'sum': 56840.183386, 'median': 0.629144, 'range': 2.0, 'percentile_5': 0.134994, 'mad': 0.188218, 'majority': -1.0, 'minority': -0.993671, 'unique': 89338, 'nodata': 85, 'valid': 0.99909}], - [{'sum': 30468.535156, 'median': 0.619463, 'range': 2.0, + [{'sum': 30468.534019, 'median': 0.619463, 'range': 2.0, 'percentile_5': 0.15824, 'mad': 0.161906, 'majority': -1.0, 'minority': -0.991903, 'unique': 49427, 'nodata': 0, 'valid': 1.0}], - [{'sum': 44829.203125, 'median': 0.704596, 'range': 2.0, + [{'sum': 44829.198908, 'median': 0.704596, 'range': 2.0, 'percentile_5': 0.307304, 'mad': 0.169161, 'majority': 1.0, 'minority': -0.969231, 'unique': 61646, 'nodata': 337, 'valid': 0.994777}], - [{'sum': 58079.574219, 'median': 0.727142, 'range': 2.0, + [{'sum': 58079.578482, 'median': 0.727142, 'range': 2.0, 'percentile_5': 0.211673, 'mad': 0.138837, 'majority': 1.0, 'minority': -0.962406, 'unique': 79181, 'nodata': 0, 'valid': 1.0}], - [{'sum': 2966.584717, 'median': 0.786099, 'range': 1.206738, + [{'sum': 2966.584778, 'median': 0.786099, 'range': 1.206738, 'percentile_5': 0.241052, 'mad': 0.153867, 'majority': 0.941176, 'minority': -0.206738, 'unique': 4007, 'nodata': 62, 'valid': 0.984878}], - [{'sum': 17599.695312, 'median': 0.586743, 'range': 1.83908, + [{'sum': 17599.695646, 'median': 0.586743, 'range': 1.83908, 'percentile_5': 0.255563, 'mad': 0.168826, 'majority': 1.0, 'minority': -0.83908, 'unique': 28731, 'nodata': 290, 'valid': 0.990177}], - [{'sum': 97063.34375, 'median': 0.548321, 'range': 2.0, + [{'sum': 97063.345854, 'median': 0.548321, 'range': 2.0, 'percentile_5': 0.121661, 'mad': 0.177506, 'majority': -1.0, 'minority': -0.996276, 'unique': 165519, 'nodata': 55, 'valid': 0.99969}], - [{'sum': 10982.208008, 'median': 0.647735, 'range': 0.961919, + [{'sum': 10982.207629, 'median': 0.647735, 'range': 0.961919, 'percentile_5': 0.135674, 'mad': 0.24678, 'majority': 1.0, 'minority': 0.038081, 'unique': 17550, 'nodata': 337, 'valid': 0.98139}], - [{'sum': 89315.601562, 'median': 0.491085, 'range': 2.0, + [{'sum': 89315.609434, 'median': 0.491085, 'range': 2.0, 'percentile_5': 0.12513, 'mad': 0.170515, 'majority': -1.0, 'minority': -0.995074, 'unique': 158615, 'nodata': 0, 'valid': 1.0}], - [{'sum': 17259.078125, 'median': 0.589402, 'range': 1.993846, + [{'sum': 17259.079321, 'median': 0.589402, 'range': 1.993846, 'percentile_5': 0.324657, 'mad': 0.142261, 'majority': -1.0, 'minority': -0.974359, 'unique': 28279, 'nodata': 0, 'valid': 1.0}], - [{'sum': 31228.535156, 'median': 0.564933, 'range': 2.0, + [{'sum': 31228.533127, 'median': 0.564933, 'range': 2.0, 'percentile_5': 0.111111, 'mad': 0.227924, 'majority': -1.0, 'minority': -0.962085, 'unique': 54423, 'nodata': 0, 'valid': 1.0}], - [{'sum': 20114.677734, 'median': 0.537082, 'range': 2.0, + [{'sum': 20114.676941, 'median': 0.537082, 'range': 2.0, 'percentile_5': 0.231518, 'mad': 0.16068, 'majority': -1.0, 'minority': -0.991091, 'unique': 34565, 'nodata': 259, 'valid': 0.992713}]] @@ -231,17 +231,17 @@ def test_compute_zonal_stats_per_category(): for geom_stats in statistics for d in geom_stats for key, val in d.items()] # ref is the following - ref = [[{'11min': 40.407368, '11max': 44.083961, '11mean': 42.293809, '11count': 244, '11std': 0.849256, - '31min': 40.224247, '31max': 68.915146, '31mean': 46.435981, '31count': 12347, '31std': 4.744863, - '32min': 38.825829, '32max': 46.798634, '32mean': 42.545211, '32count': 6657, '32std': 1.33232, - '42min': 38.214043, '42max': 59.93272, '42mean': 43.375167, '42count': 2716, '42std': 1.595453, - '43min': 38.214043, '43max': 45.618088, '43mean': 42.273938, '43count': 875, '43std': 1.241663}], - [{'11min': 38.475033, '11max': 45.613518, '11mean': 42.386634, '11count': 73437, '11std': 1.090365, + ref = [[{'11min': 40.407368, '11max': 44.083961, '11mean': 42.293813, '11count': 244, '11std': 0.849256, + '31min': 40.224247, '31max': 68.915146, '31mean': 46.435982, '31count': 12347, '31std': 4.744863, + '32min': 38.825829, '32max': 46.798634, '32mean': 42.54521, '32count': 6657, '32std': 1.33232, + '42min': 38.214043, '42max': 59.93272, '42mean': 43.375171, '42count': 2716, '42std': 1.595453, + '43min': 38.214043, '43max': 45.618088, '43mean': 42.273934, '43count': 875, '43std': 1.241663}], + [{'11min': 38.475033, '11max': 45.613518, '11mean': 42.386635, '11count': 73437, '11std': 1.090365, '12min': 39.241253, '12max': 43.433277, '12mean': 41.991684, '12count': 17339, '12std': 0.313978, '31min': 33.781662, '31max': 60.81406, '31mean': 44.562402, '31count': 12743, '31std': 3.051407, - '32min': 37.670204, '32max': 64.120644, '32mean': 44.396163, '32count': 18284, '32std': 2.749989, - '42min': 40.806831, '42max': 65.240021, '42mean': 45.256736, '42count': 45240, '42std': 3.348138, - '43min': 42.136879, '43max': 65.507011, '43mean': 48.296517, '43count': 59113, '43std': 3.671298}]] + '32min': 37.670204, '32max': 64.120644, '32mean': 44.396165, '32count': 18284, '32std': 2.749989, + '42min': 40.806831, '42max': 65.240021, '42mean': 45.256735, '42count': 45240, '42std': 3.348138, + '43min': 42.136879, '43max': 65.507011, '43mean': 48.296514, '43count': 59113, '43std': 3.671298}]] for geom_stats, ref_stats in zip(statistics, ref): for i, band in enumerate(bands): @@ -269,17 +269,17 @@ def test_compute_zonal_stats_per_category(): for geom_stats in statistics for d in geom_stats for key, val in d.items()] # ref is the following - ref = [[{'cetemin': 40.407368, 'cetemax': 44.083961, 'cetemean': 42.293809, 'cetecount': 244, 'cetestd': 0.849256, - 'feumin': 40.224247, 'feumax': 68.915146, 'feumean': 46.435981, 'feucount': 12347, 'feustd': 4.744863, - 'conmin': 38.825829, 'conmax': 46.798634, 'conmean': 42.545211, 'concount': 6657, 'constd': 1.33232, - 'udimin': 38.214043, 'udimax': 59.93272, 'udimean': 43.375167, 'udicount': 2716, 'udistd': 1.595453, - 'zicmin': 38.214043, 'zicmax': 45.618088, 'zicmean': 42.273938, 'ziccount': 875, 'zicstd': 1.241663}], - [{'cetemin': 38.475033, 'cetemax': 45.613518, 'cetemean': 42.386634, 'cetecount': 73437, 'cetestd': 1.090365, + ref = [[{'cetemin': 40.407368, 'cetemax': 44.083961, 'cetemean': 42.293813, 'cetecount': 244, 'cetestd': 0.849256, + 'feumin': 40.224247, 'feumax': 68.915146, 'feumean': 46.435982, 'feucount': 12347, 'feustd': 4.744863, + 'conmin': 38.825829, 'conmax': 46.798634, 'conmean': 42.54521, 'concount': 6657, 'constd': 1.33232, + 'udimin': 38.214043, 'udimax': 59.93272, 'udimean': 43.375171, 'udicount': 2716, 'udistd': 1.595453, + 'zicmin': 38.214043, 'zicmax': 45.618088, 'zicmean': 42.273934, 'ziccount': 875, 'zicstd': 1.241663}], + [{'cetemin': 38.475033, 'cetemax': 45.613518, 'cetemean': 42.386635, 'cetecount': 73437, 'cetestd': 1.090365, 'chivmin': 39.241253, 'chivmax': 43.433277, 'chivmean': 41.991684, 'chivcount': 17339, 'chivstd': 0.313978, 'feumin': 33.781662, 'feumax': 60.81406, 'feumean': 44.562402, 'feucount': 12743, 'feustd': 3.051407, - 'conmin': 37.670204, 'conmax': 64.120644, 'conmean': 44.396163, 'concount': 18284, 'constd': 2.749989, - 'udimin': 40.806831, 'udimax': 65.240021, 'udimean': 45.256736, 'udicount': 45240, 'udistd': 3.348138, - 'zicmin': 42.136879, 'zicmax': 65.507011, 'zicmean': 48.296517, 'ziccount': 59113, 'zicstd': 3.671298}]] + 'conmin': 37.670204, 'conmax': 64.120644, 'conmean': 44.396165, 'concount': 18284, 'constd': 2.749989, + 'udimin': 40.806831, 'udimax': 65.240021, 'udimean': 45.256735, 'udicount': 45240, 'udistd': 3.348138, + 'zicmin': 42.136879, 'zicmax': 65.507011, 'zicmean': 48.296514, 'ziccount': 59113, 'zicstd': 3.671298}]] for geom_stats, ref_stats in zip(statistics, ref): for i, band in enumerate(bands): diff --git a/tests/tests_refs/test_zonalstats/SENTINEL2A_20180928-105515-685_L2A_T30TYP_D-ndvi-stats-outliers.tif b/tests/tests_refs/test_zonalstats/SENTINEL2A_20180928-105515-685_L2A_T30TYP_D-ndvi-stats-outliers.tif index ae4b9647..146bf28d 100644 Binary files a/tests/tests_refs/test_zonalstats/SENTINEL2A_20180928-105515-685_L2A_T30TYP_D-ndvi-stats-outliers.tif and b/tests/tests_refs/test_zonalstats/SENTINEL2A_20180928-105515-685_L2A_T30TYP_D-ndvi-stats-outliers.tif differ