diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 8a554634..0a36b6ba 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -70,6 +70,3 @@ jobs: - name: Deploy to pypi uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/README.md b/README.md index 5cede088..971e8d29 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ [![Reddit](https://img.shields.io/reddit/subreddit-subscribers/TerminusDB?style=social)](https://www.reddit.com/r/TerminusDB/) [![Twitter](https://img.shields.io/twitter/follow/terminusdb?color=skyblue&label=Follow%20on%20Twitter&logo=twitter&style=flat)](https://twitter.com/TerminusDB) -[![release version](https://img.shields.io/pypi/v/terminusdb-client.svg?logo=pypi)](https://pypi.python.org/pypi/terminusdb-client/) -[![downloads](https://img.shields.io/pypi/dm/terminusdb-client.svg?logo=pypi)](https://pypi.python.org/pypi/terminusdb-client/) +[![release version](https://img.shields.io/pypi/v/terminusdb.svg?logo=pypi)](https://pypi.python.org/pypi/terminusdb/) +[![downloads](https://img.shields.io/pypi/dm/terminusdb.svg?logo=pypi)](https://pypi.python.org/pypi/terminusdb/) [![build status](https://img.shields.io/github/workflow/status/terminusdb/terminusdb-client-python/Python%20package?logo=github)](https://github.com/terminusdb/terminusdb-client-python/actions) [![documentation](https://img.shields.io/github/deployments/terminusdb/terminusdb-client-python/github-pages?label=documentation&logo=github)](https://terminusdb.org/docs/python) @@ -18,6 +18,10 @@ > Python client for TerminusDB and TerminusCMS. +> **Migrating from `terminusdb-client`?** This package was formerly known as +> `terminusdb-client`. Simply install `terminusdb` instead — both `import terminusdb` +> and `import terminusdb_client` continue to work, so no code changes are required. + [**TerminusDB**][terminusdb] is an [open-source][terminusdb-repo] graph database and document store. It allows you to link JSON documents in a powerful knowledge graph all through a simple document API, with full git-for-data version control. @@ -28,26 +32,26 @@ graph all through a simple document API, with full git-for-data version control. ## Requirements -- [TerminusDB v11.1](https://github.com/terminusdb/terminusdb-server) +- [TerminusDB v12](https://github.com/terminusdb/terminusdb-server) - [Python >=3.9](https://www.python.org/downloads) ## Release Notes and Previous Versions -TerminusDB Client v11.1 works with TerminusDB v11.1 and the [DFRNT cloud service](https://dfrnt.com). Please check the [Release Notes](RELEASE_NOTES.md) to find out what has changed. +TerminusDB Client v12 works with TerminusDB v12 onwards and the [DFRNT cloud service](https://dfrnt.com). Please check the [Release Notes](RELEASE_NOTES.md) to find out what has changed. ## Installation - TerminusDB Client can be downloaded from PyPI using pip: -`python -m pip install terminusdb-client` +`python -m pip install terminusdb` This only includes the core Python Client (Client) and WOQLQuery. If you want to use woqlDataframe or the import and export CSV function in the Scaffolding CLI tool: -`python -m pip install terminusdb-client[dataframe]` +`python -m pip install terminusdb[dataframe]` *if you are installing from `zsh` you have to quote the argument like this:* -`python -m pip install 'terminusdb-client[dataframe]'` +`python -m pip install 'terminusdb[dataframe]'` - Install from source: @@ -66,19 +70,21 @@ If you want to use woqlDataframe or the import and export CSV function in the Sc Connect to local host ```Python -from terminusdb_client import Client +from terminusdb import Client client = Client("http://127.0.0.1:6363/") client.connect() ``` +The previous import path `from terminusdb_client import Client` also continues to work. + Connect to TerminusDB in the cloud *check the documentation on the DFRNT support page about how to add your [API token](https://support.dfrnt.com/portal/en/kb/articles/api) to the environment variable* ```Python -from terminusdb_client import Client +from terminusdb import Client team="MyTeam" client = Client(f"https://studio.dfrnt.com/api/hosted/{team}/") @@ -94,7 +100,7 @@ client.create_database("MyDatabase") #### Create a schema ```Python -from terminusdb_client.schema import Schema, DocumentTemplate, RandomKey +from terminusdb.schema import Schema, DocumentTemplate, RandomKey my_schema = Schema() diff --git a/pyproject.toml b/pyproject.toml index 835107e2..254158ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,14 @@ [tool.poetry] -name = "terminusdb-client" +name = "terminusdb" version = "12.0.3" -description = "Python client for Terminus DB" -authors = ["TerminusDB group"] +description = "Terminus DB Python client" +authors = ["TerminusDB group", "DFRNT AB"] license = "Apache Software License" readme = "README.md" +packages = [ + {include = "terminusdb_client"}, + {include = "terminusdb"}, +] [tool.poetry.dependencies] python = ">=3.9.0,<3.13" diff --git a/terminusdb/__init__.py b/terminusdb/__init__.py new file mode 100644 index 00000000..85c681c2 --- /dev/null +++ b/terminusdb/__init__.py @@ -0,0 +1,4 @@ +# Re-export everything from terminusdb_client for the new import path. +# Both `import terminusdb` and `import terminusdb_client` are supported. +from terminusdb_client import * # noqa +from terminusdb_client import Client, WOQLClient, WOQLQuery, Var, Vars, Patch, GraphType, WOQLDataFrame, WOQLSchema # noqa diff --git a/terminusdb/client/__init__.py b/terminusdb/client/__init__.py new file mode 100644 index 00000000..f1d1877e --- /dev/null +++ b/terminusdb/client/__init__.py @@ -0,0 +1,2 @@ +from terminusdb_client.client import * # noqa +from terminusdb_client.client import Client, GraphType, Patch # noqa diff --git a/terminusdb/query_syntax/__init__.py b/terminusdb/query_syntax/__init__.py new file mode 100644 index 00000000..bdacc3af --- /dev/null +++ b/terminusdb/query_syntax/__init__.py @@ -0,0 +1 @@ +from terminusdb_client.query_syntax import * # noqa diff --git a/terminusdb/schema/__init__.py b/terminusdb/schema/__init__.py new file mode 100644 index 00000000..f0887c73 --- /dev/null +++ b/terminusdb/schema/__init__.py @@ -0,0 +1 @@ +from terminusdb_client.schema import * # noqa diff --git a/terminusdb_client/scripts/scripts.py b/terminusdb_client/scripts/scripts.py index 5de6f3bd..8ffb34cd 100644 --- a/terminusdb_client/scripts/scripts.py +++ b/terminusdb_client/scripts/scripts.py @@ -486,7 +486,6 @@ def importcsv( embedded = [x.lower().replace(" ", "_") for x in embedded] try: pd = import_module("pandas") - np = import_module("numpy") except ImportError: raise ImportError( "Library 'pandas' is required to import csv, either install 'pandas' or install woqlDataframe requirements as follows: python -m pip install -U terminus-client-python[dataframe]" @@ -500,6 +499,48 @@ def importcsv( # "not schema" make it always False if adding the schema option has_schema = not schema and class_name in client.get_existing_classes() + def _df_to_schema(class_name, df): + class_dict = {"@type": "Class", "@id": class_name} + for col, dtype in dict(df.dtypes).items(): + if embedded and col in embedded: + converted_type = class_name + else: + # Map pandas/numpy dtype to Python type + # Uses dtype.kind for compatibility with numpy 2.0+ and pandas 3.0+ + dtype_kind = getattr(dtype, "kind", "O") + if dtype.type is str or dtype_kind in ("U", "O", "S", "T"): + converted_type = str + elif dtype_kind in ("i", "u"): + converted_type = int + elif dtype_kind == "f": + converted_type = float + elif dtype_kind == "b": + converted_type = bool + elif dtype_kind == "M": + converted_type = dt.datetime + elif dtype_kind == "m": + converted_type = dt.timedelta + else: + converted_type = str + converted_type = wt.to_woql_type(converted_type) + + if id_ and col == id_: + class_dict[col] = converted_type + elif na == "optional" and col not in keys: + class_dict[col] = {"@type": "Optional", "@class": converted_type} + else: + class_dict[col] = converted_type + # if id_ is not None: + # pass # don't need key if id is specified + # elif keys: + # class_dict["@key"] = {"@type": "Random"} + # elif na == "optional": + # # have to use random key cause keys will be optional + # class_dict["@key"] = {"@type": "Random"} + # else: + # class_dict["@key"] = {"@type": "Random"} + return class_dict + with pd.read_csv(csv_file, sep=sep, chunksize=chunksize, dtype=dtype) as reader: for df in tqdm(reader): if any(df.isna().any()) and na == "error": @@ -512,15 +553,7 @@ def importcsv( converted_col = col.lower().replace(" ", "_").replace(".", "_") df.rename(columns={col: converted_col}, inplace=True) if not has_schema: - class_dict = _df_to_schema( - class_name, - df, - np, - embedded=embedded, - id_col=id_, - na_mode=na, - keys=keys, - ) + class_dict = _df_to_schema(class_name, df) if message is None: schema_msg = f"Schema object insert/ update with {csv_file} by Python client." else: diff --git a/terminusdb_client/tests/integration_tests/test_conftest.py b/terminusdb_client/tests/integration_tests/test_conftest.py index c12f4055..576ee2c9 100644 --- a/terminusdb_client/tests/integration_tests/test_conftest.py +++ b/terminusdb_client/tests/integration_tests/test_conftest.py @@ -14,20 +14,18 @@ class TestServerDetection: """Test server detection helper functions""" @patch("terminusdb_client.tests.integration_tests.conftest.requests.get") - def test_local_server_running_200(self, mock_get): - """Test local server detection returns True for HTTP 200""" - mock_response = Mock() - mock_response.status_code = 200 - mock_get.return_value = mock_response + def test_local_server_running_any_response(self, mock_get): + """Test local server detection returns True for any HTTP response""" + mock_get.return_value = Mock() assert is_local_server_running() is True mock_get.assert_called_once_with("http://127.0.0.1:6363/api/", timeout=2) @patch("terminusdb_client.tests.integration_tests.conftest.requests.get") - def test_local_server_running_404(self, mock_get): - """Test local server detection returns True for HTTP 404""" + def test_local_server_running_401(self, mock_get): + """Test local server detection returns True for HTTP 401 (unauthorized)""" mock_response = Mock() - mock_response.status_code = 404 + mock_response.status_code = 401 mock_get.return_value = mock_response assert is_local_server_running() is True @@ -47,20 +45,18 @@ def test_local_server_not_running_timeout(self, mock_get): assert is_local_server_running() is False @patch("terminusdb_client.tests.integration_tests.conftest.requests.get") - def test_docker_server_running_200(self, mock_get): - """Test Docker server detection returns True for HTTP 200""" - mock_response = Mock() - mock_response.status_code = 200 - mock_get.return_value = mock_response + def test_docker_server_running_any_response(self, mock_get): + """Test Docker server detection returns True for any HTTP response""" + mock_get.return_value = Mock() assert is_docker_server_running() is True mock_get.assert_called_once_with("http://127.0.0.1:6366/api/", timeout=2) @patch("terminusdb_client.tests.integration_tests.conftest.requests.get") - def test_docker_server_running_404(self, mock_get): - """Test Docker server detection returns True for HTTP 404""" + def test_docker_server_running_401(self, mock_get): + """Test Docker server detection returns True for HTTP 401 (unauthorized)""" mock_response = Mock() - mock_response.status_code = 404 + mock_response.status_code = 401 mock_get.return_value = mock_response assert is_docker_server_running() is True @@ -73,20 +69,18 @@ def test_docker_server_not_running(self, mock_get): assert is_docker_server_running() is False @patch("terminusdb_client.tests.integration_tests.conftest.requests.get") - def test_jwt_server_running_200(self, mock_get): - """Test JWT server detection returns True for HTTP 200""" - mock_response = Mock() - mock_response.status_code = 200 - mock_get.return_value = mock_response + def test_jwt_server_running_any_response(self, mock_get): + """Test JWT server detection returns True for any HTTP response""" + mock_get.return_value = Mock() assert is_jwt_server_running() is True mock_get.assert_called_once_with("http://127.0.0.1:6367/api/", timeout=2) @patch("terminusdb_client.tests.integration_tests.conftest.requests.get") - def test_jwt_server_running_404(self, mock_get): - """Test JWT server detection returns True for HTTP 404""" + def test_jwt_server_running_401(self, mock_get): + """Test JWT server detection returns True for HTTP 401 (unauthorized)""" mock_response = Mock() - mock_response.status_code = 404 + mock_response.status_code = 401 mock_get.return_value = mock_response assert is_jwt_server_running() is True diff --git a/terminusdb_client/tests/integration_tests/test_woql_collect.py b/terminusdb_client/tests/integration_tests/test_woql_collect.py new file mode 100644 index 00000000..f3905485 --- /dev/null +++ b/terminusdb_client/tests/integration_tests/test_woql_collect.py @@ -0,0 +1,142 @@ +""" +Integration tests for WOQL Collect predicate. + +Collect gathers all solutions from a sub-query into a list, +completing the list/binding symmetry alongside Member: +- Member: List -> Bindings (destructure) +- Collect: Bindings -> List (gather) +""" + +import pytest + +from terminusdb_client import Client +from terminusdb_client.woqlquery.woql_query import WOQLQuery + +test_user_agent = "terminusdb-client-python-tests" + + +def extract_values(result_list): + """Extract raw values from a list of typed literals.""" + if not result_list: + return [] + return [ + item["@value"] if isinstance(item, dict) and "@value" in item else item + for item in result_list + ] + + +class TestWOQLCollect: + """Tests for the WOQL Collect predicate.""" + + @pytest.fixture(autouse=True) + def setup_teardown(self, docker_url): + """Setup and teardown for each test.""" + self.client = Client(docker_url, user_agent=test_user_agent) + self.client.connect() + self.db_name = "test_woql_collect" + + # Create database for tests + if self.db_name in self.client.list_databases(): + self.client.delete_database(self.db_name) + self.client.create_database(self.db_name) + + # Add schema + self.client.insert_document( + [ + { + "@type": "@context", + "@base": "terminusdb:///data/", + "@schema": "terminusdb:///schema#", + }, + { + "@id": "NamedThing", + "@type": "Class", + "@key": {"@type": "Lexical", "@fields": ["name"]}, + "name": "xsd:string", + }, + ], + graph_type="schema", + full_replace=True, + ) + + # Insert test documents + self.client.insert_document( + [ + {"@type": "NamedThing", "name": "Alice"}, + {"@type": "NamedThing", "name": "Bob"}, + {"@type": "NamedThing", "name": "Carol"}, + ] + ) + + yield + + # Cleanup + self.client.delete_database(self.db_name) + + def test_collect_triple_objects_into_list(self): + """Collect gathers all matching triple objects into a single list.""" + query = WOQLQuery().collect( + "v:name", + "v:names", + WOQLQuery().triple("v:doc", "name", "v:name"), + ) + + result = self.client.query(query) + assert len(result["bindings"]) == 1 + names = sorted(extract_values(result["bindings"][0]["names"])) + assert names == ["Alice", "Bob", "Carol"] + + def test_collect_empty_result(self): + """Collect produces empty list when sub-query has no solutions.""" + query = WOQLQuery().collect( + "v:x", + "v:collected", + WOQLQuery().triple("v:doc", "nonexistent_property", "v:x"), + ) + + result = self.client.query(query) + assert len(result["bindings"]) == 1 + assert result["bindings"][0]["collected"] == [] + + def test_collect_composes_with_length(self): + """Collect result can be used with length to count solutions.""" + query = WOQLQuery().woql_and( + WOQLQuery().collect( + "v:name", + "v:names", + WOQLQuery().triple("v:doc", "name", "v:name"), + ), + WOQLQuery().length("v:names", "v:count"), + ) + + result = self.client.query(query) + assert len(result["bindings"]) == 1 + assert result["bindings"][0]["count"]["@value"] == 3 + + def test_collect_with_limit_in_subquery(self): + """Collect respects limit inside the sub-query.""" + query = WOQLQuery().collect( + "v:name", + "v:names", + WOQLQuery().limit(2, WOQLQuery().triple("v:doc", "name", "v:name")), + ) + + result = self.client.query(query) + assert len(result["bindings"]) == 1 + assert len(result["bindings"][0]["names"]) == 2 + + def test_collect_with_list_template(self): + """Collect with multi-element list template produces nested lists.""" + query = WOQLQuery().collect( + ["v:doc", "v:name"], + "v:pairs", + WOQLQuery().triple("v:doc", "name", "v:name"), + ) + + result = self.client.query(query) + assert len(result["bindings"]) == 1 + pairs = result["bindings"][0]["pairs"] + assert len(pairs) == 3 + for pair in pairs: + assert isinstance(pair, list) + assert len(pair) == 2 diff --git a/terminusdb_client/woqlquery/woql_query.py b/terminusdb_client/woqlquery/woql_query.py index 1030cdda..bb5692c5 100644 --- a/terminusdb_client/woqlquery/woql_query.py +++ b/terminusdb_client/woqlquery/woql_query.py @@ -3109,6 +3109,34 @@ def group_by(self, group_vars, template, output, groupquery=None): self._cursor["grouped"] = self._clean_object(output) return self._add_sub_query(groupquery) + def collect(self, template, into, query=None): + """Collects all solutions of a sub-query into a list. + + Completes the list/binding symmetry alongside member: + - Member: List -> Bindings (destructure) + - Collect: Bindings -> List (gather) + + Parameters + ---------- + template : str or list + A variable or list of variables specifying what to collect from each solution + into : str + Variable that will be bound to the collected list + query : WOQLQuery, optional + The query whose solutions will be collected + + Returns + ------- + WOQLQuery object + query object that can be chained and/or execute + """ + if self._cursor.get("@type"): + self._wrap_cursor_with_and() + self._cursor["@type"] = "Collect" + self._cursor["template"] = self._clean_object(template) + self._cursor["into"] = self._clean_object(into) + return self._add_sub_query(query) + def true(self): """Sets true for cursor type.