feat: use the new version of FAO GAUL (GAUL 2024)

.github/workflows/unit.yaml

@@ -7,7 +7,8 @@ on:
   pull_request:
 
 env:
-  EARTHENGINE_TOKEN: ${{ secrets.EARTHENGINE_TOKEN }}
+  EARTHENGINE_SERVICE_ACCOUNT: ${{ secrets.EARTHENGINE_SERVICE_ACCOUNT }}
+  EARTHENGINE_PROJECT: ${{ secrets.EARTHENGINE_PROJECT }}
 
 jobs:
   lint:
@@ -47,15 +48,14 @@ jobs:
   build:
     needs: [lint, mypy]
     strategy:
-      fail-fast: true
       matrix:
         os: [ubuntu-latest]
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
         include:
           - os: macos-latest # macos test
-            python-version: "3.11"
+            python-version: "3.12"
           - os: windows-latest # windows test
-            python-version: "3.11"
+            python-version: "3.12"
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3

.pre-commit-config.yaml

@@ -38,12 +38,6 @@ repos:
       - id: doc8
         stages: [commit]
 
-  - repo: https://github.com/FHPythonUtils/LicenseCheck
-    rev: "2023.1.1"
-    hooks:
-      - id: licensecheck
-        stages: [commit]
-
   - repo: https://github.com/codespell-project/codespell
     rev: v2.2.4
     hooks:

docs/conf.py

@@ -6,13 +6,7 @@
 """
 
 # -- Path setup ----------------------------------------------------------------
-import os
-import re
 from datetime import datetime
-from pathlib import Path
-
-import ee
-import httplib2
 
 # -- Project information -------------------------------------------------------
 project = "pyGAUL"
@@ -69,38 +63,3 @@
 autoapi_dirs = ["../pygaul"]
 autoapi_python_class_content = "init"
 autoapi_member_order = "groupwise"
-
-# -- Script to authenticate to Earthengine using a token -----------------------
-def gee_configure() -> None:
-    """Initialize earth engine according to the environment.
-
-    It will use the creddential file if the EARTHENGINE_TOKEN env variable exist.
-    Otherwise it use the simple Initialize command (asking the user to register if necessary).
-    """
-    # only do the initialization if the credential are missing
-    if not ee.data._credentials:
-
-        # if the credentials token is asved in the environment use it
-        if "EARTHENGINE_TOKEN" in os.environ:
-
-            # get the token from environment variable
-            ee_token = os.environ["EARTHENGINE_TOKEN"]
-
-            # as long as RDT quote the token, we need to remove the quotes before writing
-            # the string to the file
-            pattern = r"^'[^']*'$"
-            if re.match(pattern, ee_token) is not None:
-                ee_token = ee_token[1:-1]
-
-            # write the token to the appropriate folder
-            credential_folder_path = Path.home() / ".config" / "earthengine"
-            credential_folder_path.mkdir(parents=True, exist_ok=True)
-            credential_file_path = credential_folder_path / "credentials"
-            credential_file_path.write_text(ee_token)
-
-        # if the user is in local development the authentication should
-        # already be available
-        ee.Initialize(http_transport=httplib2.Http())
-
-
-gee_configure()

docs/usage.rst

@@ -25,10 +25,10 @@ For example to extract the France geometry you can use the following code:
 .. jupyter-execute::
 
     import pygaul
+    from pygaul import utils
     from geemap import Map
-    import ee
 
-    ee.Initialize()
+    utils.initialize_documentation()
 
     fc = pygaul.Items(name="France")
 
@@ -42,12 +42,12 @@ If you know the code of the area you try to use, you can use the GADM code inste
 .. jupyter-execute::
 
     import pygaul
+    from pygaul import utils
     from geemap import Map
-    import ee
 
-    ee.Initialize()
+    utils.initialize_documentation()
 
-    fc = pygaul.Items(admin="85")
+    fc = pygaul.Items(admin="301")
 
     # display it in a map
     m = Map(zoom=5, center=[46.21, 2.21])
@@ -62,10 +62,10 @@ One is not bind to only request a country, any level can be accessed using both
 .. jupyter-execute::
 
     import pygaul
+    from pygaul import utils
     from geemap import Map
-    import ee
 
-    ee.Initialize()
+    utils.initialize_documentation()
 
     fc = pygaul.Items(name="Corse-du-Sud")
 
@@ -86,12 +86,12 @@ Using the :code:`content_level` option, one can require smaller administrative l
 .. jupyter-execute::
 
     import pygaul
+    from pygaul import utils
     from geemap import Map
-    import ee
 
-    ee.Initialize()
+    utils.initialize_documentation()
 
-    fc = pygaul.Items(admin="85", content_level=2)
+    fc = pygaul.Items(admin="301", content_level=2)
 
     # display it in a map
     m = Map(zoom=5, center=[46.21, 2.21])
@@ -106,10 +106,10 @@ To perform regional analysis that aggregate multiple boundaries, you can now req
 .. jupyter-execute::
 
     import pygaul
+    from pygaul import utils
     from geemap import Map
-    import ee
 
-    ee.Initialize()
+    utils.initialize_documentation()
 
     fc = pygaul.Items(name=["France", "Germany"], content_level=1)
 
@@ -134,12 +134,12 @@ It's possible to request all countries from one single continent using one of th
 .. jupyter-execute::
 
     import pygaul
+    from pygaul import utils
     from geemap import Map
-    import ee
 
-    ee.Initialize()
+    utils.initialize_documentation()
 
-    fc = pygaul.Items(name="europe")
+    fc = pygaul.Items(name="Europe")
 
     # display it in a map
     m = Map(zoom=4, center = [49.38237278700955, 31.464843750000004])
@@ -157,7 +157,7 @@ For example to get the names and codes of all the departments in France you can
 
     import pygaul
 
-    pygaul.Names(admin="85", content_level=2)
+    pygaul.Names(admin="301", content_level=2) # france
 
 .. note::
 
@@ -167,7 +167,7 @@ For example to get the names and codes of all the departments in France you can
 
         import pygaul
 
-        pygaul.Names(admin="1270", content_level=2, complete=True)
+        pygaul.Names(name="Auvergne-Rhône-Alpes", content_level=2, complete=True)
 
 .. note::
 
@@ -188,9 +188,9 @@ If you make an error when writing the name of your input, the error message will
     :raises: ValueError
 
     import pygaul
-    import ee
+    from pygaul import utils
 
-    ee.Initialize()
+    utils.initialize_documentation()
 
     fc = pygaul.Items(name="Franc")
 

pygaul/__init__.py

@@ -3,7 +3,6 @@
 
 This lib provides access to FAO GAUL 2015 datasets from a Python script. it is the best boundary dataset available for GEE at this point. We provide access to The current version (2015) administrative areas till level 2.
 """
-import json
 import warnings
 from difflib import get_close_matches
 from functools import lru_cache
@@ -21,8 +20,7 @@
 __email__ = "pierrick.rambaud49@gmail.com"
 
 __gaul_data__ = Path(__file__).parent / "data" / "gaul_database.parquet"
-__gaul_continent__ = Path(__file__).parent / "data" / "gaul_continent.json"
-__gaul_asset__ = "FAO/GAUL/2015/level{}"
+__gaul_asset__ = "projects/sat-io/open-datasets/FAO/GAUL/GAUL_2024_L{}"
 
 
 @lru_cache(maxsize=1)
@@ -63,7 +61,7 @@ def __init__(
             id = name if name else admin
 
             # read the data and find if the element exist
-            column = "ADM{}_NAME" if is_name else "ADM{}_CODE"
+            column = "gaul{}_name" if is_name else "gaul{}_code"
             is_in = (
                 df.filter([column.format(i) for i in range(3)])
                 .apply(lambda col: col.str.lower())
@@ -82,18 +80,18 @@ def __init__(
                 else:
                     close_ids = [i.upper() for i in close_ids]
                 raise ValueError(
-                    f'The requested "{id}" is not part of FAO GAUL 2015. The closest '
+                    f'The requested "{id}" is not part of FAO GAUL 2024. The closest '
                     f'matches are: {", ".join(close_ids)}.'
                 )
 
             # Get the code of the associated country of the identifed area and the associated level
             line = is_in[~((~is_in).all(axis=1))].idxmax(1)
-            level = line.iloc[0][3]
+            level = line.iloc[0][4]
 
             # load the max_level available in the requested area
             sub_df = df[df[column.format(level)].str.fullmatch(id, case=False)]
             max_level = next(
-                i for i in reversed(range(3)) if (sub_df[f"ADM{i}_NAME"] != "").any()
+                i for i in reversed(range(3)) if (sub_df[f"gaul{i}_name"] != "").any()
             )
 
             # get the request level from user
@@ -119,7 +117,7 @@ def __init__(
             content_level = 0 if content_level == -1 else content_level
 
         # get the columns name corresponding to the requested level
-        columns = [f"ADM{content_level}_NAME", f"ADM{content_level}_CODE"]
+        columns = [f"gaul{content_level}_name", f"gaul{content_level}_code"]
 
         # the list will contain duplicate as all the smaller admin level will be included
         sub_df = sub_df.drop_duplicates(subset=columns, ignore_index=True)
@@ -160,10 +158,10 @@ def __init__(
         if names == [""] == admins:
             raise ValueError('at least "name" or "admin" need to be set.')
 
-        # special parsing for continents. They are saved as admins to avoid any duplication
-        continents = json.loads(__gaul_continent__.read_text())
-        if len(names) == 1 and names[0].lower() in continents:
-            admins = [c for c in continents[names[0].lower()]]
+        # special parsing for continents. They are associated to the countries by FAO.
+        continents = _df().continent.unique()
+        if len(names) == 1 and (c := names[0].lower()) in continents:
+            admins = [a for a in _df()[_df().continent == c].gaul0_code.unique()]
             names = [""]
 
         # use itertools, normally one of them is empty so it will raise an error
@@ -201,20 +199,20 @@ def _items(
                 f"If you don't know the GAUL code, use the following code, "
                 f'it will return the GAUL codes as well:\n`Names(name="{name}")`'
             )
-        df.columns[0][3]
+        df.columns[0][4]
 
         # now load the useful one to get content_level
         df = Names(name, admin, content_level)
-        content_level = df.columns[1][3]
+        content_level = df.columns[1][4]
 
         # checks have already been performed in Names and there should
         # be one single result
-        ids = [int(v) for v in df[f"ADM{content_level}_CODE"].to_list()]
+        ids = [int(v) for v in df[f"gaul{content_level}_code"].to_list()]
 
         # read the accurate dataset
         feature_collection = ee.FeatureCollection(
             __gaul_asset__.format(content_level)
-        ).filter(ee.Filter.inList(f"ADM{content_level}_CODE", ids))
+        ).filter(ee.Filter.inList(f"gaul{content_level}_code", ids))
 
         return feature_collection