From 63f269f1df3e6f3de09ce6da738febff0e022814 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 18 Oct 2024 22:34:34 +0100 Subject: [PATCH 01/29] improve arrow function --- piccolo/columns/column_types.py | 33 +++------------------------------ piccolo/querystring.py | 6 ++++++ 2 files changed, 9 insertions(+), 30 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index d16329b49..229295c66 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2352,40 +2352,13 @@ class JSONB(JSON): def column_type(self): return "JSONB" # Must be defined, we override column_type() in JSON() - def arrow(self, key: str) -> JSONB: + def arrow(self, key: str) -> QueryString: """ Allows part of the JSON structure to be returned - for example, for {"a": 1}, and a key value of "a", then 1 will be returned. """ - instance = t.cast(JSONB, self.copy()) - instance.json_operator = f"-> '{key}'" - return instance - - def get_select_string( - self, engine_type: str, with_alias: bool = True - ) -> QueryString: - select_string = self._meta.get_full_name(with_alias=False) - - if self.json_operator is not None: - select_string += f" {self.json_operator}" - - if with_alias: - alias = self._alias or self._meta.get_default_alias() - select_string += f' AS "{alias}"' - - return QueryString(select_string) - - def eq(self, value) -> Where: - """ - See ``Boolean.eq`` for more details. - """ - return self.__eq__(value) - - def ne(self, value) -> Where: - """ - See ``Boolean.ne`` for more details. - """ - return self.__ne__(value) + alias = self._alias or self._meta.get_default_alias() + return QueryString("{} -> {}", self, key, alias=alias) ########################################################################### # Descriptors diff --git a/piccolo/querystring.py b/piccolo/querystring.py index 22f5f215a..ae8748117 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -264,6 +264,12 @@ def __eq__(self, value) -> QueryString: # type: ignore[override] def __ne__(self, value) -> QueryString: # type: ignore[override] return QueryString("{} != {}", self, value) + def eq(self, value) -> QueryString: + return self.__eq__(value) + + def ne(self, value) -> QueryString: + return self.__ne__(value) + def __add__(self, value) -> QueryString: return QueryString("{} + {}", self, value) From a58663b7bc9312c1357ba9d659dddf909b1b3bc9 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sat, 19 Oct 2024 22:24:29 +0100 Subject: [PATCH 02/29] fix tests --- piccolo/columns/column_types.py | 31 +++++++++++++++++++++++++++++-- piccolo/query/base.py | 18 +++++++++++------- piccolo/querystring.py | 10 ++++++++-- 3 files changed, 48 insertions(+), 11 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 229295c66..c7c910199 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2335,6 +2335,33 @@ def __set__(self, obj, value: t.Union[str, t.Dict]): obj.__dict__[self._meta.name] = value +class JSONQueryString(QueryString): + """ + Functionally this is basically the same as ``QueryString``, we just need + ``Query._process_results`` to be able to differentiate it from a normal + ``QueryString`` just incase the user specified ``.output(load_json=True)``. + """ + + def clean_value(self, value: t.Any): + if not isinstance(value, (str, QueryString)): + value = dump_json(value) + return value + + def __eq__(self, value) -> QueryString: # type: ignore[override] + value = self.clean_value(value) + return QueryString("{} = {}", self, value) + + def __ne__(self, value) -> QueryString: # type: ignore[override] + value = self.clean_value(value) + return QueryString("{} != {}", self, value) + + def eq(self, value) -> QueryString: + return self.__eq__(value) + + def ne(self, value) -> QueryString: + return self.__ne__(value) + + class JSONB(JSON): """ Used for storing JSON strings - Postgres only. The data is stored in a @@ -2352,13 +2379,13 @@ class JSONB(JSON): def column_type(self): return "JSONB" # Must be defined, we override column_type() in JSON() - def arrow(self, key: str) -> QueryString: + def arrow(self, key: str) -> JSONQueryString: """ Allows part of the JSON structure to be returned - for example, for {"a": 1}, and a key value of "a", then 1 will be returned. """ alias = self._alias or self._meta.get_default_alias() - return QueryString("{} -> {}", self, key, alias=alias) + return JSONQueryString("{} -> {}", self, key, alias=alias) ########################################################################### # Descriptors diff --git a/piccolo/query/base.py b/piccolo/query/base.py index ff49c0e3b..f01c12e23 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -3,7 +3,7 @@ import typing as t from time import time -from piccolo.columns.column_types import JSON, JSONB +from piccolo.columns.column_types import JSON, JSONB, JSONQueryString from piccolo.custom_types import QueryResponseType, TableInstance from piccolo.query.mixins import ColumnsDelegate from piccolo.querystring import QueryString @@ -65,16 +65,20 @@ async def _process_results(self, results) -> QueryResponseType: self, "columns_delegate", None ) + json_column_names: t.List[str] = [] + if columns_delegate is not None: - json_columns = [ - i - for i in columns_delegate.selected_columns - if isinstance(i, (JSON, JSONB)) - ] + json_columns: t.List[t.Union[JSON, JSONB]] = [] + + for column in columns_delegate.selected_columns: + if isinstance(column, (JSON, JSONB)): + json_columns.append(column) + elif isinstance(column, JSONQueryString): + if alias := column._alias: + json_column_names.append(alias) else: json_columns = self.table._meta.json_columns - json_column_names = [] for column in json_columns: if column._alias is not None: json_column_names.append(column._alias) diff --git a/piccolo/querystring.py b/piccolo/querystring.py index ae8748117..7dec758a8 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -259,10 +259,16 @@ def get_where_string(self, engine_type: str) -> QueryString: # Basic logic def __eq__(self, value) -> QueryString: # type: ignore[override] - return QueryString("{} = {}", self, value) + if value is None: + return QueryString("{} IS NULL", self) + else: + return QueryString("{} = {}", self, value) def __ne__(self, value) -> QueryString: # type: ignore[override] - return QueryString("{} != {}", self, value) + if value is None: + return QueryString("{} IS NOT NULL", self, value) + else: + return QueryString("{} != {}", self, value) def eq(self, value) -> QueryString: return self.__eq__(value) From 976f3efe399ff5848fc227b3e4bc9d16a3adc422 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 20 Oct 2024 07:34:57 +0100 Subject: [PATCH 03/29] refactor, so we have an `Arrow` function --- piccolo/columns/column_types.py | 34 +++++----------------------- piccolo/query/base.py | 5 +++-- piccolo/query/functions/json.py | 40 +++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 31 deletions(-) create mode 100644 piccolo/query/functions/json.py diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index c7c910199..c6f46ad5b 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -70,6 +70,7 @@ class Band(Table): if t.TYPE_CHECKING: # pragma: no cover from piccolo.columns.base import ColumnMeta + from piccolo.query.functions.json import Arrow from piccolo.table import Table @@ -2335,33 +2336,6 @@ def __set__(self, obj, value: t.Union[str, t.Dict]): obj.__dict__[self._meta.name] = value -class JSONQueryString(QueryString): - """ - Functionally this is basically the same as ``QueryString``, we just need - ``Query._process_results`` to be able to differentiate it from a normal - ``QueryString`` just incase the user specified ``.output(load_json=True)``. - """ - - def clean_value(self, value: t.Any): - if not isinstance(value, (str, QueryString)): - value = dump_json(value) - return value - - def __eq__(self, value) -> QueryString: # type: ignore[override] - value = self.clean_value(value) - return QueryString("{} = {}", self, value) - - def __ne__(self, value) -> QueryString: # type: ignore[override] - value = self.clean_value(value) - return QueryString("{} != {}", self, value) - - def eq(self, value) -> QueryString: - return self.__eq__(value) - - def ne(self, value) -> QueryString: - return self.__ne__(value) - - class JSONB(JSON): """ Used for storing JSON strings - Postgres only. The data is stored in a @@ -2379,13 +2353,15 @@ class JSONB(JSON): def column_type(self): return "JSONB" # Must be defined, we override column_type() in JSON() - def arrow(self, key: str) -> JSONQueryString: + def arrow(self, key: str) -> Arrow: """ Allows part of the JSON structure to be returned - for example, for {"a": 1}, and a key value of "a", then 1 will be returned. """ + from piccolo.query.functions.json import Arrow + alias = self._alias or self._meta.get_default_alias() - return JSONQueryString("{} -> {}", self, key, alias=alias) + return Arrow(column=self, key=key, alias=alias) ########################################################################### # Descriptors diff --git a/piccolo/query/base.py b/piccolo/query/base.py index f01c12e23..9f3e8874f 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -3,8 +3,9 @@ import typing as t from time import time -from piccolo.columns.column_types import JSON, JSONB, JSONQueryString +from piccolo.columns.column_types import JSON, JSONB from piccolo.custom_types import QueryResponseType, TableInstance +from piccolo.query.functions.json import Arrow from piccolo.query.mixins import ColumnsDelegate from piccolo.querystring import QueryString from piccolo.utils.encoding import load_json @@ -73,7 +74,7 @@ async def _process_results(self, results) -> QueryResponseType: for column in columns_delegate.selected_columns: if isinstance(column, (JSON, JSONB)): json_columns.append(column) - elif isinstance(column, JSONQueryString): + elif isinstance(column, Arrow): if alias := column._alias: json_column_names.append(alias) else: diff --git a/piccolo/query/functions/json.py b/piccolo/query/functions/json.py new file mode 100644 index 000000000..b4c9f4d56 --- /dev/null +++ b/piccolo/query/functions/json.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import typing as t + +from piccolo.querystring import QueryString +from piccolo.utils.encoding import dump_json + +if t.TYPE_CHECKING: + from piccolo.columns.column_types import JSONB + + +class Arrow(QueryString): + """ + Functionally this is basically the same as ``QueryString``, we just need + ``Query._process_results`` to be able to differentiate it from a normal + ``QueryString`` just in case the user specified + ``.output(load_json=True)``. + """ + + def __init__(self, column: JSONB, key: str, alias: t.Optional[str] = None): + super().__init__("{} -> {}", column, key, alias=alias) + + def clean_value(self, value: t.Any): + if not isinstance(value, (str, QueryString)): + value = dump_json(value) + return value + + def __eq__(self, value) -> QueryString: # type: ignore[override] + value = self.clean_value(value) + return QueryString("{} = {}", self, value) + + def __ne__(self, value) -> QueryString: # type: ignore[override] + value = self.clean_value(value) + return QueryString("{} != {}", self, value) + + def eq(self, value) -> QueryString: + return self.__eq__(value) + + def ne(self, value) -> QueryString: + return self.__ne__(value) From fe2ce3ddf0290524e28795b7e63a9ac412ad0a56 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 20 Oct 2024 07:54:40 +0100 Subject: [PATCH 04/29] add test for nested arrow functions --- piccolo/query/functions/__init__.py | 2 ++ piccolo/query/functions/json.py | 7 ++++++- tests/query/functions/test_json.py | 25 +++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 tests/query/functions/test_json.py diff --git a/piccolo/query/functions/__init__.py b/piccolo/query/functions/__init__.py index 3163f6d1c..7d51dca21 100644 --- a/piccolo/query/functions/__init__.py +++ b/piccolo/query/functions/__init__.py @@ -1,11 +1,13 @@ from .aggregate import Avg, Count, Max, Min, Sum from .datetime import Day, Extract, Hour, Month, Second, Strftime, Year +from .json import Arrow from .math import Abs, Ceil, Floor, Round from .string import Concat, Length, Lower, Ltrim, Reverse, Rtrim, Upper from .type_conversion import Cast __all__ = ( "Abs", + "Arrow", "Avg", "Cast", "Ceil", diff --git a/piccolo/query/functions/json.py b/piccolo/query/functions/json.py index b4c9f4d56..4805698db 100644 --- a/piccolo/query/functions/json.py +++ b/piccolo/query/functions/json.py @@ -17,7 +17,12 @@ class Arrow(QueryString): ``.output(load_json=True)``. """ - def __init__(self, column: JSONB, key: str, alias: t.Optional[str] = None): + def __init__( + self, + column: JSONB | QueryString, + key: str, + alias: t.Optional[str] = None, + ): super().__init__("{} -> {}", column, key, alias=alias) def clean_value(self, value: t.Any): diff --git a/tests/query/functions/test_json.py b/tests/query/functions/test_json.py new file mode 100644 index 000000000..3ec3c4c38 --- /dev/null +++ b/tests/query/functions/test_json.py @@ -0,0 +1,25 @@ +from piccolo.columns import JSONB +from piccolo.query.functions.json import Arrow +from piccolo.table import Table +from piccolo.testing.test_case import AsyncTableTest + + +class RecordingStudio(Table): + facilities = JSONB(null=True) + + +class TestArrow(AsyncTableTest): + + tables = [RecordingStudio] + + async def test_nested(self): + await RecordingStudio( + {RecordingStudio.facilities: {"a": {"b": {"c": 1}}}} + ).save() + + response = await RecordingStudio.select( + Arrow(Arrow(RecordingStudio.facilities, "a"), "b").as_alias( + "b_value" + ) + ) + self.assertListEqual(response, [{"b_value": '{"c": 1}'}]) From 64a208ebbb52220015d423d9994326d06488f911 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 20 Oct 2024 09:50:53 +0100 Subject: [PATCH 05/29] skip sqlite --- tests/query/functions/test_json.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/query/functions/test_json.py b/tests/query/functions/test_json.py index 3ec3c4c38..b039a706a 100644 --- a/tests/query/functions/test_json.py +++ b/tests/query/functions/test_json.py @@ -2,12 +2,14 @@ from piccolo.query.functions.json import Arrow from piccolo.table import Table from piccolo.testing.test_case import AsyncTableTest +from tests.base import engines_skip class RecordingStudio(Table): facilities = JSONB(null=True) +@engines_skip("sqlite") class TestArrow(AsyncTableTest): tables = [RecordingStudio] From 91c3f84e8d96418e53cc1f387bff272bc7014251 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 20 Oct 2024 12:29:06 +0100 Subject: [PATCH 06/29] allow `arrow` to access keys multiple levels deep --- piccolo/columns/column_types.py | 16 ++++++++++++++-- piccolo/query/functions/json.py | 4 ++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index c6f46ad5b..dd36fd985 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2356,12 +2356,24 @@ def column_type(self): def arrow(self, key: str) -> Arrow: """ Allows part of the JSON structure to be returned - for example, - for {"a": 1}, and a key value of "a", then 1 will be returned. + for ``{"a": 1}``, and a key value of ``"a"``, then 1 will be returned. + + It can be used multiple levels deep:: + + >>> await RecordingStudio.select( + ... RecordingStudio.facilities.arrow("facilities.guitars") + ... ) + """ from piccolo.query.functions.json import Arrow alias = self._alias or self._meta.get_default_alias() - return Arrow(column=self, key=key, alias=alias) + + key_elements = key.split(".") + output = Arrow(self, key=key_elements[0], alias=alias) + for key_element in key_elements[1:]: + output = Arrow(output, key=key_element, alias=alias) + return output ########################################################################### # Descriptors diff --git a/piccolo/query/functions/json.py b/piccolo/query/functions/json.py index 4805698db..968c2eb4c 100644 --- a/piccolo/query/functions/json.py +++ b/piccolo/query/functions/json.py @@ -19,11 +19,11 @@ class Arrow(QueryString): def __init__( self, - column: JSONB | QueryString, + identifier: JSONB | QueryString, key: str, alias: t.Optional[str] = None, ): - super().__init__("{} -> {}", column, key, alias=alias) + super().__init__("{} -> {}", identifier, key, alias=alias) def clean_value(self, value: t.Any): if not isinstance(value, (str, QueryString)): From 326a15dfde60a2746951dddafbe37b1f29dd4ead Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Sun, 20 Oct 2024 12:29:26 +0100 Subject: [PATCH 07/29] improve the example data in the playground for JSON data --- piccolo/apps/playground/commands/run.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/piccolo/apps/playground/commands/run.py b/piccolo/apps/playground/commands/run.py index 3f435c44b..6a14d76e4 100644 --- a/piccolo/apps/playground/commands/run.py +++ b/piccolo/apps/playground/commands/run.py @@ -233,6 +233,7 @@ def populate(): RecordingStudio.facilities: { "restaurant": True, "mixing_desk": True, + "instruments": {"electric_guitar": 10, "drum_kit": 2}, }, } ) @@ -244,6 +245,7 @@ def populate(): RecordingStudio.facilities: { "restaurant": False, "mixing_desk": True, + "instruments": {"electric_guitar": 6, "drum_kit": 3}, }, }, ) From 7493d3e1f95424682ffb90fdc4d7cce7b9832ea5 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 11:58:45 +0100 Subject: [PATCH 08/29] move arrow function to JSON, as it can be used by JSON or JSONB --- piccolo/columns/column_types.py | 47 +++++++++++++++------------------ 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index dd36fd985..12e097016 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2320,6 +2320,25 @@ def column_type(self): else: return "JSON" + ########################################################################### + + def arrow(self, key: t.Union[str, int]) -> Arrow: + """ + Allows part of the JSON structure to be returned - for example, + for ``{"a": 1}``, and a key value of ``"a"``, then 1 will be returned. + + It can be used multiple levels deep:: + + >>> await RecordingStudio.select( + ... RecordingStudio.facilities.arrow("facilities.guitars") + ... ) + + """ + from piccolo.query.functions.json import Arrow + + alias = self._alias or self._meta.get_default_alias() + return Arrow(identifier=self, key=key, alias=alias) + ########################################################################### # Descriptors @@ -2339,9 +2358,9 @@ def __set__(self, obj, value: t.Union[str, t.Dict]): class JSONB(JSON): """ Used for storing JSON strings - Postgres only. The data is stored in a - binary format, and can be queried. Insertion can be slower (as it needs to - be converted to the binary format). The benefits of JSONB generally - outweigh the downsides. + binary format, and can be queried more efficiently. Insertion can be slower + (as it needs to be converted to the binary format). The benefits of JSONB + generally outweigh the downsides. :param default: Either a JSON string can be provided, or a Python ``dict`` or ``list`` @@ -2353,28 +2372,6 @@ class JSONB(JSON): def column_type(self): return "JSONB" # Must be defined, we override column_type() in JSON() - def arrow(self, key: str) -> Arrow: - """ - Allows part of the JSON structure to be returned - for example, - for ``{"a": 1}``, and a key value of ``"a"``, then 1 will be returned. - - It can be used multiple levels deep:: - - >>> await RecordingStudio.select( - ... RecordingStudio.facilities.arrow("facilities.guitars") - ... ) - - """ - from piccolo.query.functions.json import Arrow - - alias = self._alias or self._meta.get_default_alias() - - key_elements = key.split(".") - output = Arrow(self, key=key_elements[0], alias=alias) - for key_element in key_elements[1:]: - output = Arrow(output, key=key_element, alias=alias) - return output - ########################################################################### # Descriptors From b22073fe02774554682fd8b2a4486f9d44cb041e Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 11:59:22 +0100 Subject: [PATCH 09/29] add `arrow` function to Arrow, so it can be called recursively --- piccolo/query/functions/json.py | 41 +++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/piccolo/query/functions/json.py b/piccolo/query/functions/json.py index 968c2eb4c..2d91b3cc9 100644 --- a/piccolo/query/functions/json.py +++ b/piccolo/query/functions/json.py @@ -6,21 +6,26 @@ from piccolo.utils.encoding import dump_json if t.TYPE_CHECKING: - from piccolo.columns.column_types import JSONB + from piccolo.columns.column_types import JSON class Arrow(QueryString): """ - Functionally this is basically the same as ``QueryString``, we just need - ``Query._process_results`` to be able to differentiate it from a normal - ``QueryString`` just in case the user specified - ``.output(load_json=True)``. + Allows you to drill into a JSON object. + + Arrow isn't really a function - it's an operator (i.e. ``->``), but for + Piccolo's purposes it works basically the same. + + In the future we might move this to a different folder. For that reason, + don't use it directly - use the arrow function on ``JSON`` and ``JSONB`` + columns. + """ def __init__( self, - identifier: JSONB | QueryString, - key: str, + identifier: t.Union[JSON, QueryString], + key: t.Union[str, int], alias: t.Optional[str] = None, ): super().__init__("{} -> {}", identifier, key, alias=alias) @@ -43,3 +48,25 @@ def eq(self, value) -> QueryString: def ne(self, value) -> QueryString: return self.__ne__(value) + + def arrow(self, key: t.Union[str, int]) -> Arrow: + """ + This allows you to drill multiple levels deep into a JSON object. + + For example:: + + >>> await RecordingStudio.select( + ... RecordingStudio.name, + ... RecordingStudio.facilities.arrow( + ... "instruments" + ... ).arrow( + ... "drum_kit" + ... ).as_alias("drum_kit") + ... ).output(load_json=True) + [ + {'name': 'Abbey Road', 'drum_kit': 2}, + {'name': 'Electric Lady', 'drum_kit': 3} + ] + + """ + return Arrow(identifier=self, key=key, alias=self._alias) From b51d07ef0088b279fa3ac742ca095fcb41925184 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 11:59:54 +0100 Subject: [PATCH 10/29] change heading levels of JSON docs --- docs/src/piccolo/schema/column_types.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/src/piccolo/schema/column_types.rst b/docs/src/piccolo/schema/column_types.rst index 37484978a..5b106eeed 100644 --- a/docs/src/piccolo/schema/column_types.rst +++ b/docs/src/piccolo/schema/column_types.rst @@ -201,6 +201,7 @@ JSONB .. autoclass:: JSONB +=========== Serialising =========== @@ -224,6 +225,7 @@ You can also pass in a JSON string if you prefer: ) await studio.save() +============= Deserialising ============= @@ -257,11 +259,12 @@ With ``objects`` queries, we can modify the returned JSON, and then save it: studio['facilities']['restaurant'] = False await studio.save() +===== arrow ===== -``JSONB`` columns have an ``arrow`` function, which is useful for retrieving -a subset of the JSON data: +``JSON`` and ``JSONB`` columns have an ``arrow`` function, which is useful for +retrieving a subset of the JSON data: .. code-block:: python @@ -271,6 +274,8 @@ a subset of the JSON data: ... ).output(load_json=True) [{'name': 'Abbey Road', 'mixing_desk': True}] +.. note:: Postgres and CockroachDB only. + It can also be used for filtering in a where clause: .. code-block:: python @@ -280,6 +285,7 @@ It can also be used for filtering in a where clause: ... ) [{'name': 'Abbey Road'}] +============= Handling null ============= From d89900b2c93190d0a044245e503f1fef126e7212 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 12:14:37 +0100 Subject: [PATCH 11/29] move `Arrow` to operators folder --- piccolo/apps/playground/commands/run.py | 4 ++-- piccolo/columns/column_types.py | 4 ++-- piccolo/query/base.py | 2 +- piccolo/query/functions/__init__.py | 2 -- piccolo/query/operators/__init__.py | 0 piccolo/query/{functions => operators}/json.py | 13 ++++--------- tests/query/operators/__init__.py | 0 tests/query/{functions => operators}/test_json.py | 2 +- 8 files changed, 10 insertions(+), 17 deletions(-) create mode 100644 piccolo/query/operators/__init__.py rename piccolo/query/{functions => operators}/json.py (79%) create mode 100644 tests/query/operators/__init__.py rename tests/query/{functions => operators}/test_json.py (93%) diff --git a/piccolo/apps/playground/commands/run.py b/piccolo/apps/playground/commands/run.py index 6a14d76e4..ba6ce6983 100644 --- a/piccolo/apps/playground/commands/run.py +++ b/piccolo/apps/playground/commands/run.py @@ -233,7 +233,7 @@ def populate(): RecordingStudio.facilities: { "restaurant": True, "mixing_desk": True, - "instruments": {"electric_guitar": 10, "drum_kit": 2}, + "instruments": {"electric_guitars": 10, "drum_kits": 2}, }, } ) @@ -245,7 +245,7 @@ def populate(): RecordingStudio.facilities: { "restaurant": False, "mixing_desk": True, - "instruments": {"electric_guitar": 6, "drum_kit": 3}, + "instruments": {"electric_guitars": 6, "drum_kits": 3}, }, }, ) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 12e097016..399008342 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -70,7 +70,7 @@ class Band(Table): if t.TYPE_CHECKING: # pragma: no cover from piccolo.columns.base import ColumnMeta - from piccolo.query.functions.json import Arrow + from piccolo.query.operators.json import Arrow from piccolo.table import Table @@ -2334,7 +2334,7 @@ def arrow(self, key: t.Union[str, int]) -> Arrow: ... ) """ - from piccolo.query.functions.json import Arrow + from piccolo.query.operators.json import Arrow alias = self._alias or self._meta.get_default_alias() return Arrow(identifier=self, key=key, alias=alias) diff --git a/piccolo/query/base.py b/piccolo/query/base.py index 9f3e8874f..abb46e2a2 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -5,7 +5,7 @@ from piccolo.columns.column_types import JSON, JSONB from piccolo.custom_types import QueryResponseType, TableInstance -from piccolo.query.functions.json import Arrow +from piccolo.query.operators.json import Arrow from piccolo.query.mixins import ColumnsDelegate from piccolo.querystring import QueryString from piccolo.utils.encoding import load_json diff --git a/piccolo/query/functions/__init__.py b/piccolo/query/functions/__init__.py index 7d51dca21..3163f6d1c 100644 --- a/piccolo/query/functions/__init__.py +++ b/piccolo/query/functions/__init__.py @@ -1,13 +1,11 @@ from .aggregate import Avg, Count, Max, Min, Sum from .datetime import Day, Extract, Hour, Month, Second, Strftime, Year -from .json import Arrow from .math import Abs, Ceil, Floor, Round from .string import Concat, Length, Lower, Ltrim, Reverse, Rtrim, Upper from .type_conversion import Cast __all__ = ( "Abs", - "Arrow", "Avg", "Cast", "Ceil", diff --git a/piccolo/query/operators/__init__.py b/piccolo/query/operators/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/piccolo/query/functions/json.py b/piccolo/query/operators/json.py similarity index 79% rename from piccolo/query/functions/json.py rename to piccolo/query/operators/json.py index 2d91b3cc9..d46c097d5 100644 --- a/piccolo/query/functions/json.py +++ b/piccolo/query/operators/json.py @@ -13,12 +13,7 @@ class Arrow(QueryString): """ Allows you to drill into a JSON object. - Arrow isn't really a function - it's an operator (i.e. ``->``), but for - Piccolo's purposes it works basically the same. - - In the future we might move this to a different folder. For that reason, - don't use it directly - use the arrow function on ``JSON`` and ``JSONB`` - columns. + You can access via the arrow function on ``JSON`` and ``JSONB`` columns. """ @@ -61,11 +56,11 @@ def arrow(self, key: t.Union[str, int]) -> Arrow: ... "instruments" ... ).arrow( ... "drum_kit" - ... ).as_alias("drum_kit") + ... ).as_alias("drum_kits") ... ).output(load_json=True) [ - {'name': 'Abbey Road', 'drum_kit': 2}, - {'name': 'Electric Lady', 'drum_kit': 3} + {'name': 'Abbey Road', 'drum_kits': 2}, + {'name': 'Electric Lady', 'drum_kits': 3} ] """ diff --git a/tests/query/operators/__init__.py b/tests/query/operators/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/query/functions/test_json.py b/tests/query/operators/test_json.py similarity index 93% rename from tests/query/functions/test_json.py rename to tests/query/operators/test_json.py index b039a706a..924f0cb13 100644 --- a/tests/query/functions/test_json.py +++ b/tests/query/operators/test_json.py @@ -1,5 +1,5 @@ from piccolo.columns import JSONB -from piccolo.query.functions.json import Arrow +from piccolo.query.operators.json import Arrow from piccolo.table import Table from piccolo.testing.test_case import AsyncTableTest from tests.base import engines_skip From f1e31997869c4dba0893ddbddc2fd7b0498b99e4 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 15:03:21 +0100 Subject: [PATCH 12/29] update docs --- docs/src/piccolo/schema/column_types.rst | 90 +++++++++++++++++++----- 1 file changed, 72 insertions(+), 18 deletions(-) diff --git a/docs/src/piccolo/schema/column_types.rst b/docs/src/piccolo/schema/column_types.rst index 5b106eeed..7838337c2 100644 --- a/docs/src/piccolo/schema/column_types.rst +++ b/docs/src/piccolo/schema/column_types.rst @@ -189,16 +189,12 @@ Storing JSON can be useful in certain situations, for example - raw API responses, data from a Javascript app, and for storing data with an unknown or changing schema. -==== -JSON -==== +==================== +``JSON`` / ``JSONB`` +==================== .. autoclass:: JSON -===== -JSONB -===== - .. autoclass:: JSONB =========== @@ -259,29 +255,87 @@ With ``objects`` queries, we can modify the returned JSON, and then save it: studio['facilities']['restaurant'] = False await studio.save() -===== -arrow -===== +========= +``arrow`` +========= + +``JSON`` and ``JSONB`` columns have an ``arrow`` operator, which is useful for +retrieving a subset of the JSON data. + +.. note:: Postgres and CockroachDB only. + +``select`` queries +================== + +If we have the following JSON stored in the ``RecordingStudio.facility`` +column: + +.. code-block:: json + + { + "instruments": { + "drum_kits": 2, + "electric_guitars": 10 + }, + "restaurant": true, + "technicians": [ + { + "name": "Alice Jones" + }, + { + "name": "Bob Williams" + } + ] + } -``JSON`` and ``JSONB`` columns have an ``arrow`` function, which is useful for -retrieving a subset of the JSON data: +We can retrieve the ``restaurant`` value from the JSON object: .. code-block:: python >>> await RecordingStudio.select( - ... RecordingStudio.name, - ... RecordingStudio.facilities.arrow('mixing_desk').as_alias('mixing_desk') + ... RecordingStudio.facilities.arrow('restaurant') + ... .as_alias('restaurant') ... ).output(load_json=True) - [{'name': 'Abbey Road', 'mixing_desk': True}] + [{'restaurant': True}, ...] -.. note:: Postgres and CockroachDB only. +You can drill multiple levels deep by calling ``arrow`` multiple times. + +Here we fetch the number of drum kits that the recording studio has: + +.. code-block:: python + + >>> await RecordingStudio.select( + ... RecordingStudio.facilities.arrow("instruments") + ... .arrow("drum_kits") + ... .as_alias("drum_kits") + ... ).output(load_json=True) + [{'drum_kits': 2}, ...] + +If you have a JSON object which consists of arrays and objects, then you can +navigate the array elements by passing in an integer to ``arrow``. + +Here we fetch the first technician from the array: + +.. code-block:: python + + >>> await RecordingStudio.select( + ... RecordingStudio.facilities.arrow("technicians") + ... .arrow(0) + ... .arrow("name") + ... .as_alias("technician_name") + ... ).output(load_json=True) + + [{'technician_name': 'Alice Jones'}, ...] + +``where`` clauses +================= -It can also be used for filtering in a where clause: +The ``arrow`` operator can also be used for filtering in a where clause: .. code-block:: python >>> await RecordingStudio.select(RecordingStudio.name).where( - ... RecordingStudio.facilities.arrow('mixing_desk') == True + ... RecordingStudio.facilities.arrow('mixing_desk').eq(True) ... ) [{'name': 'Abbey Road'}] From 71f279aa4fe5d1cf580bebdae7c31b580b6a360d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 15:03:32 +0100 Subject: [PATCH 13/29] improve docstring --- piccolo/columns/column_types.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 399008342..0b2b87098 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2357,10 +2357,10 @@ def __set__(self, obj, value: t.Union[str, t.Dict]): class JSONB(JSON): """ - Used for storing JSON strings - Postgres only. The data is stored in a - binary format, and can be queried more efficiently. Insertion can be slower - (as it needs to be converted to the binary format). The benefits of JSONB - generally outweigh the downsides. + Used for storing JSON strings - Postgres / CochroachDB only. The data is + stored in a binary format, and can be queried more efficiently. Insertion + can be slower (as it needs to be converted to the binary format). The + benefits of JSONB generally outweigh the downsides. :param default: Either a JSON string can be provided, or a Python ``dict`` or ``list`` From e2bce7fe49019a6c97e1ed330b5aebcfdf15bd90 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 15:03:48 +0100 Subject: [PATCH 14/29] add `technicians` to example JSON --- piccolo/apps/playground/commands/run.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/piccolo/apps/playground/commands/run.py b/piccolo/apps/playground/commands/run.py index ba6ce6983..343c02e38 100644 --- a/piccolo/apps/playground/commands/run.py +++ b/piccolo/apps/playground/commands/run.py @@ -234,6 +234,10 @@ def populate(): "restaurant": True, "mixing_desk": True, "instruments": {"electric_guitars": 10, "drum_kits": 2}, + "technicians": [ + {"name": "Alice Jones"}, + {"name": "Bob Williams"}, + ], }, } ) @@ -246,6 +250,9 @@ def populate(): "restaurant": False, "mixing_desk": True, "instruments": {"electric_guitars": 6, "drum_kits": 3}, + "technicians": [ + {"name": "Frank Smith"}, + ], }, }, ) From 27f6711fd82dfd3dd5914125530d0c6df50ea441 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 15:40:01 +0100 Subject: [PATCH 15/29] improve docstrings --- piccolo/query/operators/json.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/piccolo/query/operators/json.py b/piccolo/query/operators/json.py index d46c097d5..2a77c51fb 100644 --- a/piccolo/query/operators/json.py +++ b/piccolo/query/operators/json.py @@ -13,7 +13,8 @@ class Arrow(QueryString): """ Allows you to drill into a JSON object. - You can access via the arrow function on ``JSON`` and ``JSONB`` columns. + You can access this via the ``arrow`` function on ``JSON`` and ``JSONB`` + columns. """ @@ -56,6 +57,7 @@ def arrow(self, key: t.Union[str, int]) -> Arrow: ... "instruments" ... ).arrow( ... "drum_kit" + ... "drum_kits" ... ).as_alias("drum_kits") ... ).output(load_json=True) [ From 203190db0243e59eb4ee36800146592ec0fa6e15 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 15:40:37 +0100 Subject: [PATCH 16/29] allow `QueryString` as an arg type to `Arrow` --- piccolo/query/operators/json.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piccolo/query/operators/json.py b/piccolo/query/operators/json.py index 2a77c51fb..bab3c186a 100644 --- a/piccolo/query/operators/json.py +++ b/piccolo/query/operators/json.py @@ -21,7 +21,7 @@ class Arrow(QueryString): def __init__( self, identifier: t.Union[JSON, QueryString], - key: t.Union[str, int], + key: t.Union[str, int, QueryString], alias: t.Optional[str] = None, ): super().__init__("{} -> {}", identifier, key, alias=alias) @@ -45,7 +45,7 @@ def eq(self, value) -> QueryString: def ne(self, value) -> QueryString: return self.__ne__(value) - def arrow(self, key: t.Union[str, int]) -> Arrow: + def arrow(self, key: t.Union[str, int, QueryString]) -> Arrow: """ This allows you to drill multiple levels deep into a JSON object. From c1622ca2e3a35346b64199f769b2852e24110b4a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 15:41:02 +0100 Subject: [PATCH 17/29] fix docstring error --- piccolo/query/operators/json.py | 1 - 1 file changed, 1 deletion(-) diff --git a/piccolo/query/operators/json.py b/piccolo/query/operators/json.py index bab3c186a..5f1d7e255 100644 --- a/piccolo/query/operators/json.py +++ b/piccolo/query/operators/json.py @@ -56,7 +56,6 @@ def arrow(self, key: t.Union[str, int, QueryString]) -> Arrow: ... RecordingStudio.facilities.arrow( ... "instruments" ... ).arrow( - ... "drum_kit" ... "drum_kits" ... ).as_alias("drum_kits") ... ).output(load_json=True) From 7ee664dddcb9367b9f0443fc0e5053c2375bc905 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 15:41:14 +0100 Subject: [PATCH 18/29] make sure integers can be passed in --- piccolo/query/operators/json.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/piccolo/query/operators/json.py b/piccolo/query/operators/json.py index 5f1d7e255..b08dfeeb2 100644 --- a/piccolo/query/operators/json.py +++ b/piccolo/query/operators/json.py @@ -24,6 +24,11 @@ def __init__( key: t.Union[str, int, QueryString], alias: t.Optional[str] = None, ): + if isinstance(key, int): + # asyncpg only accepts integer keys if we explicitly mark it as an + # int. + key = QueryString("{}::int", key) + super().__init__("{} -> {}", identifier, key, alias=alias) def clean_value(self, value: t.Any): From b46ab42f7010ffa335e3b05439774f61dd62130b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 15:41:32 +0100 Subject: [PATCH 19/29] add `QueryString` as an arg type to `arrow` method --- piccolo/columns/column_types.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 0b2b87098..aa69b0827 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2322,15 +2322,12 @@ def column_type(self): ########################################################################### - def arrow(self, key: t.Union[str, int]) -> Arrow: + def arrow(self, key: t.Union[str, int, QueryString]) -> Arrow: """ - Allows part of the JSON structure to be returned - for example, - for ``{"a": 1}``, and a key value of ``"a"``, then 1 will be returned. - - It can be used multiple levels deep:: + Allows part of the JSON structure to be returned - for example:: >>> await RecordingStudio.select( - ... RecordingStudio.facilities.arrow("facilities.guitars") + ... RecordingStudio.facilities.arrow("restaurant") ... ) """ From fb3e13cfc6184f23b730e438f526d02313ae1092 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 16:58:52 +0100 Subject: [PATCH 20/29] added `GetElementFromPath` --- piccolo/columns/column_types.py | 49 +++++++++++++++++-- piccolo/query/base.py | 4 +- piccolo/query/operators/json.py | 76 ++++++++++++++++++++---------- tests/query/operators/test_json.py | 8 ++-- 4 files changed, 102 insertions(+), 35 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index aa69b0827..83ca9ce49 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -70,7 +70,10 @@ class Band(Table): if t.TYPE_CHECKING: # pragma: no cover from piccolo.columns.base import ColumnMeta - from piccolo.query.operators.json import Arrow + from piccolo.query.operators.json import ( + GetChildElement, + GetElementFromPath, + ) from piccolo.table import Table @@ -2322,19 +2325,55 @@ def column_type(self): ########################################################################### - def arrow(self, key: t.Union[str, int, QueryString]) -> Arrow: + def arrow(self, key: t.Union[str, int, QueryString]) -> GetChildElement: """ - Allows part of the JSON structure to be returned - for example:: + Allows a child element of the JSON structure to be returned - for + example:: >>> await RecordingStudio.select( ... RecordingStudio.facilities.arrow("restaurant") ... ) """ - from piccolo.query.operators.json import Arrow + from piccolo.query.operators.json import GetChildElement + + alias = self._alias or self._meta.get_default_alias() + return GetChildElement(identifier=self, key=key, alias=alias) + + def from_path( + self, + path: t.List[t.Union[str, int]], + ) -> GetElementFromPath: + """ + Allows an element of the JSON structure to be returned, which can be + arbitrarily deep. For example:: + + >>> await RecordingStudio.select( + ... RecordingStudio.facilities.from_path([ + ... "technician", + ... 0, + ... "first_name" + ... ]) + ... ) + + It's the same as calling ``arrow`` multiple times, but is more + efficient / convenient if extracting highly nested data:: + + >>> await RecordingStudio.select( + ... RecordingStudio.facilities.arrow( + ... "technician" + ... ).arrow( + ... 0 + ... ).arrow( + ... "first_name" + ... ) + ... ) + + """ + from piccolo.query.operators.json import GetElementFromPath alias = self._alias or self._meta.get_default_alias() - return Arrow(identifier=self, key=key, alias=alias) + return GetElementFromPath(identifier=self, path=path, alias=alias) ########################################################################### # Descriptors diff --git a/piccolo/query/base.py b/piccolo/query/base.py index abb46e2a2..45049e1e1 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -5,8 +5,8 @@ from piccolo.columns.column_types import JSON, JSONB from piccolo.custom_types import QueryResponseType, TableInstance -from piccolo.query.operators.json import Arrow from piccolo.query.mixins import ColumnsDelegate +from piccolo.query.operators.json import JSONQueryString from piccolo.querystring import QueryString from piccolo.utils.encoding import load_json from piccolo.utils.objects import make_nested_object @@ -74,7 +74,7 @@ async def _process_results(self, results) -> QueryResponseType: for column in columns_delegate.selected_columns: if isinstance(column, (JSON, JSONB)): json_columns.append(column) - elif isinstance(column, Arrow): + elif isinstance(column, JSONQueryString): if alias := column._alias: json_column_names.append(alias) else: diff --git a/piccolo/query/operators/json.py b/piccolo/query/operators/json.py index b08dfeeb2..19adafde9 100644 --- a/piccolo/query/operators/json.py +++ b/piccolo/query/operators/json.py @@ -9,27 +9,7 @@ from piccolo.columns.column_types import JSON -class Arrow(QueryString): - """ - Allows you to drill into a JSON object. - - You can access this via the ``arrow`` function on ``JSON`` and ``JSONB`` - columns. - - """ - - def __init__( - self, - identifier: t.Union[JSON, QueryString], - key: t.Union[str, int, QueryString], - alias: t.Optional[str] = None, - ): - if isinstance(key, int): - # asyncpg only accepts integer keys if we explicitly mark it as an - # int. - key = QueryString("{}::int", key) - - super().__init__("{} -> {}", identifier, key, alias=alias) +class JSONQueryString(QueryString): def clean_value(self, value: t.Any): if not isinstance(value, (str, QueryString)): @@ -50,9 +30,33 @@ def eq(self, value) -> QueryString: def ne(self, value) -> QueryString: return self.__ne__(value) - def arrow(self, key: t.Union[str, int, QueryString]) -> Arrow: + +class GetChildElement(QueryString): + """ + Allows you to get a child element from a JSON object. + + You can access this via the ``arrow`` function on ``JSON`` and ``JSONB`` + columns. + + """ + + def __init__( + self, + identifier: t.Union[JSON, QueryString], + key: t.Union[str, int, QueryString], + alias: t.Optional[str] = None, + ): + if isinstance(key, int): + # asyncpg only accepts integer keys if we explicitly mark it as an + # int. + key = QueryString("{}::int", key) + + super().__init__("{} -> {}", identifier, key, alias=alias) + + def arrow(self, key: t.Union[str, int, QueryString]) -> GetChildElement: """ - This allows you to drill multiple levels deep into a JSON object. + This allows you to drill multiple levels deep into a JSON object if + needed. For example:: @@ -70,4 +74,28 @@ def arrow(self, key: t.Union[str, int, QueryString]) -> Arrow: ] """ - return Arrow(identifier=self, key=key, alias=self._alias) + return GetChildElement(identifier=self, key=key, alias=self._alias) + + +class GetElementFromPath(JSONQueryString): + """ + Allows you to retrieve an element from a JSON object by specifying a path. + It can be several levels deep. + + You can access this via the ``from_path`` function on ``JSON`` and + ``JSONB`` columns. + + """ + + def __init__( + self, + identifier: t.Union[JSON, QueryString], + path: t.List[t.Union[str, int]], + alias: t.Optional[str] = None, + ): + super().__init__( + "{} #> {}", + identifier, + [str(i) if isinstance(i, int) else i for i in path], + alias=alias, + ) diff --git a/tests/query/operators/test_json.py b/tests/query/operators/test_json.py index 924f0cb13..a6bb9a6ca 100644 --- a/tests/query/operators/test_json.py +++ b/tests/query/operators/test_json.py @@ -1,5 +1,5 @@ from piccolo.columns import JSONB -from piccolo.query.operators.json import Arrow +from piccolo.query.operators.json import GetChildElement from piccolo.table import Table from piccolo.testing.test_case import AsyncTableTest from tests.base import engines_skip @@ -20,8 +20,8 @@ async def test_nested(self): ).save() response = await RecordingStudio.select( - Arrow(Arrow(RecordingStudio.facilities, "a"), "b").as_alias( - "b_value" - ) + GetChildElement( + GetChildElement(RecordingStudio.facilities, "a"), "b" + ).as_alias("b_value") ) self.assertListEqual(response, [{"b_value": '{"c": 1}'}]) From 056a4c98d8783f7148e97ae74404a5bbe3a6928d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 17:12:11 +0100 Subject: [PATCH 21/29] add docs for ``from_path`` --- docs/src/piccolo/schema/column_types.rst | 26 ++++++++++++++++++++++-- piccolo/query/operators/json.py | 5 +++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/docs/src/piccolo/schema/column_types.rst b/docs/src/piccolo/schema/column_types.rst index 7838337c2..35290e176 100644 --- a/docs/src/piccolo/schema/column_types.rst +++ b/docs/src/piccolo/schema/column_types.rst @@ -260,7 +260,7 @@ With ``objects`` queries, we can modify the returned JSON, and then save it: ========= ``JSON`` and ``JSONB`` columns have an ``arrow`` operator, which is useful for -retrieving a subset of the JSON data. +retrieving a child element from the JSON data. .. note:: Postgres and CockroachDB only. @@ -298,7 +298,8 @@ We can retrieve the ``restaurant`` value from the JSON object: ... ).output(load_json=True) [{'restaurant': True}, ...] -You can drill multiple levels deep by calling ``arrow`` multiple times. +You can drill multiple levels deep by calling ``arrow`` multiple times (or +alternatively use the :ref:`from_path` method - see below). Here we fetch the number of drum kits that the recording studio has: @@ -339,6 +340,27 @@ The ``arrow`` operator can also be used for filtering in a where clause: ... ) [{'name': 'Abbey Road'}] +.. _from_path: + +============= +``from_path`` +============= + +This works the same as ``arrow`` but is more optimised if you need to return +part of a highly nested JSON structure. + +.. code-block:: python + + >>> await RecordingStudio.select( + ... RecordingStudio.facilities.from_path([ + ... "technicians", + ... 0, + ... "name" + ... ]).as_alias("technician_name") + ... ).output(load_json=True) + + [{'technician_name': 'Alice Jones'}, ...] + ============= Handling null ============= diff --git a/piccolo/query/operators/json.py b/piccolo/query/operators/json.py index 19adafde9..5f3d535d1 100644 --- a/piccolo/query/operators/json.py +++ b/piccolo/query/operators/json.py @@ -93,6 +93,11 @@ def __init__( path: t.List[t.Union[str, int]], alias: t.Optional[str] = None, ): + """ + :param path: + For example: ``["technician", 0, "name"]``. + + """ super().__init__( "{} #> {}", identifier, From 0bc8ffe45a8392771866d30897a2fbeebf8fa2e6 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 17:27:35 +0100 Subject: [PATCH 22/29] add `__getitem__` as a shortcut for the arrow method --- piccolo/columns/column_types.py | 18 ++++++++++++++++++ piccolo/query/operators/json.py | 5 +++++ 2 files changed, 23 insertions(+) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 83ca9ce49..5d0b04acc 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -2340,6 +2340,24 @@ def arrow(self, key: t.Union[str, int, QueryString]) -> GetChildElement: alias = self._alias or self._meta.get_default_alias() return GetChildElement(identifier=self, key=key, alias=alias) + def __getitem__( + self, value: t.Union[str, int, QueryString] + ) -> GetChildElement: + """ + A shortcut for the ``arrow`` method, used for retrieving a child + element. + + For example: + + .. code-block:: python + + >>> await RecordingStudio.select( + ... RecordingStudio.facilities["restaurant"] + ... ) + + """ + return self.arrow(key=value) + def from_path( self, path: t.List[t.Union[str, int]], diff --git a/piccolo/query/operators/json.py b/piccolo/query/operators/json.py index 5f3d535d1..60b9b6ed1 100644 --- a/piccolo/query/operators/json.py +++ b/piccolo/query/operators/json.py @@ -76,6 +76,11 @@ def arrow(self, key: t.Union[str, int, QueryString]) -> GetChildElement: """ return GetChildElement(identifier=self, key=key, alias=self._alias) + def __getitem__( + self, value: t.Union[str, int, QueryString] + ) -> GetChildElement: + return GetChildElement(identifier=self, key=value, alias=self._alias) + class GetElementFromPath(JSONQueryString): """ From ec6b6af93b2620ba13155454f22a86f488b159fb Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 19:38:08 +0100 Subject: [PATCH 23/29] update the docs to use the square bracket notation --- docs/src/piccolo/schema/column_types.rst | 28 +++++++++++++++--------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/docs/src/piccolo/schema/column_types.rst b/docs/src/piccolo/schema/column_types.rst index 35290e176..58b5752d2 100644 --- a/docs/src/piccolo/schema/column_types.rst +++ b/docs/src/piccolo/schema/column_types.rst @@ -255,11 +255,11 @@ With ``objects`` queries, we can modify the returned JSON, and then save it: studio['facilities']['restaurant'] = False await studio.save() -========= -``arrow`` -========= +================ +Getting elements +================ -``JSON`` and ``JSONB`` columns have an ``arrow`` operator, which is useful for +``JSON`` and ``JSONB`` columns have an ``arrow`` method, which is useful for retrieving a child element from the JSON data. .. note:: Postgres and CockroachDB only. @@ -298,6 +298,17 @@ We can retrieve the ``restaurant`` value from the JSON object: ... ).output(load_json=True) [{'restaurant': True}, ...] +As a convenience, you can just use square brackets, instead of calling +``arrow`` explicitly: + +.. code-block:: python + + >>> await RecordingStudio.select( + ... RecordingStudio.facilities['restaurant'] + ... .as_alias('restaurant') + ... ).output(load_json=True) + [{'restaurant': True}, ...] + You can drill multiple levels deep by calling ``arrow`` multiple times (or alternatively use the :ref:`from_path` method - see below). @@ -306,8 +317,7 @@ Here we fetch the number of drum kits that the recording studio has: .. code-block:: python >>> await RecordingStudio.select( - ... RecordingStudio.facilities.arrow("instruments") - ... .arrow("drum_kits") + ... RecordingStudio.facilities["instruments"]["drum_kits"] ... .as_alias("drum_kits") ... ).output(load_json=True) [{'drum_kits': 2}, ...] @@ -320,9 +330,7 @@ Here we fetch the first technician from the array: .. code-block:: python >>> await RecordingStudio.select( - ... RecordingStudio.facilities.arrow("technicians") - ... .arrow(0) - ... .arrow("name") + ... RecordingStudio.facilities["technicians"][0]["name"] ... .as_alias("technician_name") ... ).output(load_json=True) @@ -336,7 +344,7 @@ The ``arrow`` operator can also be used for filtering in a where clause: .. code-block:: python >>> await RecordingStudio.select(RecordingStudio.name).where( - ... RecordingStudio.facilities.arrow('mixing_desk').eq(True) + ... RecordingStudio.facilities['mixing_desk'].eq(True) ... ) [{'name': 'Abbey Road'}] From cedb3fed2a8753af489a52ad02ae272a99ab2b72 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 19:47:36 +0100 Subject: [PATCH 24/29] explain why the method is called `arrow` --- docs/src/piccolo/schema/column_types.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/src/piccolo/schema/column_types.rst b/docs/src/piccolo/schema/column_types.rst index 58b5752d2..e5b1f389e 100644 --- a/docs/src/piccolo/schema/column_types.rst +++ b/docs/src/piccolo/schema/column_types.rst @@ -259,8 +259,9 @@ With ``objects`` queries, we can modify the returned JSON, and then save it: Getting elements ================ -``JSON`` and ``JSONB`` columns have an ``arrow`` method, which is useful for -retrieving a child element from the JSON data. +``JSON`` and ``JSONB`` columns have an ``arrow`` method (representing the +``->`` operator in Postgres), which is useful for retrieving a child element +from the JSON data. .. note:: Postgres and CockroachDB only. From 2867fdecf3105055dc248290f31be27f5f4a9ad4 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 20:14:13 +0100 Subject: [PATCH 25/29] move arrow tests into separate class --- piccolo/query/operators/json.py | 2 +- tests/columns/test_jsonb.py | 92 ++++++++++++++++++--------------- 2 files changed, 50 insertions(+), 44 deletions(-) diff --git a/piccolo/query/operators/json.py b/piccolo/query/operators/json.py index 60b9b6ed1..ea6d05097 100644 --- a/piccolo/query/operators/json.py +++ b/piccolo/query/operators/json.py @@ -31,7 +31,7 @@ def ne(self, value) -> QueryString: return self.__ne__(value) -class GetChildElement(QueryString): +class GetChildElement(JSONQueryString): """ Allows you to get a child element from a JSON object. diff --git a/tests/columns/test_jsonb.py b/tests/columns/test_jsonb.py index fe90e769b..aabda521a 100644 --- a/tests/columns/test_jsonb.py +++ b/tests/columns/test_jsonb.py @@ -1,6 +1,6 @@ from piccolo.columns.column_types import JSONB, ForeignKey, Varchar from piccolo.table import Table -from piccolo.testing.test_case import TableTest +from piccolo.testing.test_case import AsyncTableTest, TableTest from tests.base import engines_only, engines_skip @@ -137,93 +137,99 @@ def test_as_alias_join(self): [{"name": "Guitar", "studio_facilities": {"mixing_desk": True}}], ) - def test_arrow(self): + +@engines_only("postgres", "cockroach") +class TestArrow(AsyncTableTest): + tables = [RecordingStudio, Instrument] + + async def insert_row(self): + await RecordingStudio( + name="Abbey Road", facilities='{"mixing_desk": true}' + ).save() + + async def test_arrow(self): """ Test using the arrow function to retrieve a subset of the JSON. """ - RecordingStudio( - name="Abbey Road", facilities='{"mixing_desk": true}' - ).save().run_sync() + await self.insert_row() - row = ( - RecordingStudio.select( - RecordingStudio.facilities.arrow("mixing_desk") - ) - .first() - .run_sync() - ) + row = await RecordingStudio.select( + RecordingStudio.facilities.arrow("mixing_desk") + ).first() assert row is not None self.assertEqual(row["facilities"], "true") - row = ( + row = await ( RecordingStudio.select( RecordingStudio.facilities.arrow("mixing_desk") ) .output(load_json=True) .first() - .run_sync() ) assert row is not None self.assertEqual(row["facilities"], True) - def test_arrow_as_alias(self): + async def test_arrow_as_alias(self): """ Test using the arrow function to retrieve a subset of the JSON. """ - RecordingStudio( - name="Abbey Road", facilities='{"mixing_desk": true}' - ).save().run_sync() + await self.insert_row() - row = ( - RecordingStudio.select( - RecordingStudio.facilities.arrow("mixing_desk").as_alias( - "mixing_desk" - ) + row = await RecordingStudio.select( + RecordingStudio.facilities.arrow("mixing_desk").as_alias( + "mixing_desk" ) - .first() - .run_sync() - ) + ).first() + assert row is not None + self.assertEqual(row["mixing_desk"], "true") + + async def test_square_brackets(self): + """ + Make sure we can use square brackets instead of calling ``arrow`` + explicitly. + """ + await self.insert_row() + + row = await RecordingStudio.select( + RecordingStudio.facilities["mixing_desk"].as_alias("mixing_desk") + ).first() assert row is not None self.assertEqual(row["mixing_desk"], "true") - def test_arrow_where(self): + async def test_arrow_where(self): """ Make sure the arrow function can be used within a WHERE clause. """ - RecordingStudio( - name="Abbey Road", facilities='{"mixing_desk": true}' - ).save().run_sync() + await self.insert_row() self.assertEqual( - RecordingStudio.count() - .where(RecordingStudio.facilities.arrow("mixing_desk").eq(True)) - .run_sync(), + await RecordingStudio.count().where( + RecordingStudio.facilities.arrow("mixing_desk").eq(True) + ), 1, ) self.assertEqual( - RecordingStudio.count() - .where(RecordingStudio.facilities.arrow("mixing_desk").eq(False)) - .run_sync(), + await RecordingStudio.count().where( + RecordingStudio.facilities.arrow("mixing_desk").eq(False) + ), 0, ) - def test_arrow_first(self): + async def test_arrow_first(self): """ Make sure the arrow function can be used with the first clause. """ - RecordingStudio.insert( + await RecordingStudio.insert( RecordingStudio(facilities='{"mixing_desk": true}'), RecordingStudio(facilities='{"mixing_desk": false}'), - ).run_sync() + ) self.assertEqual( - RecordingStudio.select( + await RecordingStudio.select( RecordingStudio.facilities.arrow("mixing_desk").as_alias( "mixing_desk" ) - ) - .first() - .run_sync(), + ).first(), {"mixing_desk": "true"}, ) From 4769caf55ba73e4e4036106cd91236abad1d53d9 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 20:28:59 +0100 Subject: [PATCH 26/29] add `test_multiple_levels_deep` --- tests/columns/test_jsonb.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/columns/test_jsonb.py b/tests/columns/test_jsonb.py index aabda521a..b1190f16d 100644 --- a/tests/columns/test_jsonb.py +++ b/tests/columns/test_jsonb.py @@ -196,6 +196,29 @@ async def test_square_brackets(self): assert row is not None self.assertEqual(row["mixing_desk"], "true") + async def test_multiple_levels_deep(self): + """ + Make sure elements can be extracted multiple levels deep, and using + array indexes. + """ + await RecordingStudio( + name="Abbey Road", + facilities={ + "technicians": [ + {"name": "Alice Jones"}, + {"name": "Bob Williams"}, + ] + }, + ).save() + + response = await RecordingStudio.select( + RecordingStudio.facilities["technicians"][0]["name"].as_alias( + "technician_name" + ) + ).output(load_json=True) + assert response is not None + self.assertListEqual(response, [{"technician_name": "Alice Jones"}]) + async def test_arrow_where(self): """ Make sure the arrow function can be used within a WHERE clause. From 55dee3d4ec37e2b6df9fe971bb18976ecdcc55c7 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 20:32:17 +0100 Subject: [PATCH 27/29] add tests for `for_path` --- tests/columns/test_jsonb.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/columns/test_jsonb.py b/tests/columns/test_jsonb.py index b1190f16d..f38c0de05 100644 --- a/tests/columns/test_jsonb.py +++ b/tests/columns/test_jsonb.py @@ -256,3 +256,31 @@ async def test_arrow_first(self): ).first(), {"mixing_desk": "true"}, ) + + +@engines_only("postgres", "cockroach") +class TestFromPath(AsyncTableTest): + + tables = [RecordingStudio, Instrument] + + async def test_from_path(self): + """ + Make sure ``from_path`` can be used for complex nested data. + """ + await RecordingStudio( + name="Abbey Road", + facilities={ + "technicians": [ + {"name": "Alice Jones"}, + {"name": "Bob Williams"}, + ] + }, + ).save() + + response = await RecordingStudio.select( + RecordingStudio.facilities.from_path( + ["technicians", 0, "name"] + ).as_alias("technician_name") + ).output(load_json=True) + assert response is not None + self.assertListEqual(response, [{"technician_name": "Alice Jones"}]) From 0b760e20dc752f48a78646c0b8fea4c134ba5993 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 20:36:16 +0100 Subject: [PATCH 28/29] last documentation tweaks --- docs/src/piccolo/schema/column_types.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/piccolo/schema/column_types.rst b/docs/src/piccolo/schema/column_types.rst index e5b1f389e..5c70dd482 100644 --- a/docs/src/piccolo/schema/column_types.rst +++ b/docs/src/piccolo/schema/column_types.rst @@ -268,7 +268,7 @@ from the JSON data. ``select`` queries ================== -If we have the following JSON stored in the ``RecordingStudio.facility`` +If we have the following JSON stored in the ``RecordingStudio.facilities`` column: .. code-block:: json @@ -299,8 +299,8 @@ We can retrieve the ``restaurant`` value from the JSON object: ... ).output(load_json=True) [{'restaurant': True}, ...] -As a convenience, you can just use square brackets, instead of calling -``arrow`` explicitly: +As a convenience, you can use square brackets, instead of calling ``arrow`` +explicitly: .. code-block:: python From 87f84110ba15ab8a93c04fdd95f69904ba9f3273 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 23 Oct 2024 20:46:51 +0100 Subject: [PATCH 29/29] add basic operator tests --- tests/query/operators/test_json.py | 51 ++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/tests/query/operators/test_json.py b/tests/query/operators/test_json.py index a6bb9a6ca..d7840ef9b 100644 --- a/tests/query/operators/test_json.py +++ b/tests/query/operators/test_json.py @@ -1,7 +1,8 @@ +from unittest import TestCase + from piccolo.columns import JSONB -from piccolo.query.operators.json import GetChildElement +from piccolo.query.operators.json import GetChildElement, GetElementFromPath from piccolo.table import Table -from piccolo.testing.test_case import AsyncTableTest from tests.base import engines_skip @@ -10,18 +11,42 @@ class RecordingStudio(Table): @engines_skip("sqlite") -class TestArrow(AsyncTableTest): +class TestGetChildElement(TestCase): + + def test_query(self): + """ + Make sure the generated SQL looks correct. + """ + querystring = GetChildElement( + GetChildElement(RecordingStudio.facilities, "a"), "b" + ) + + sql, query_args = querystring.compile_string() + + self.assertEqual( + sql, + '"recording_studio"."facilities" -> $1 -> $2', + ) + + self.assertListEqual(query_args, ["a", "b"]) + + +@engines_skip("sqlite") +class TestGetElementFromPath(TestCase): - tables = [RecordingStudio] + def test_query(self): + """ + Make sure the generated SQL looks correct. + """ + querystring = GetElementFromPath( + RecordingStudio.facilities, ["a", "b"] + ) - async def test_nested(self): - await RecordingStudio( - {RecordingStudio.facilities: {"a": {"b": {"c": 1}}}} - ).save() + sql, query_args = querystring.compile_string() - response = await RecordingStudio.select( - GetChildElement( - GetChildElement(RecordingStudio.facilities, "a"), "b" - ).as_alias("b_value") + self.assertEqual( + sql, + '"recording_studio"."facilities" #> $1', ) - self.assertListEqual(response, [{"b_value": '{"c": 1}'}]) + + self.assertListEqual(query_args, [["a", "b"]])