Merge pull request #63 from fredludlow/filter-operators-rebase

Filter operators rebase

README.md

@@ -386,6 +386,19 @@ Comment.smart_query(
 > ```
 > See [this example](examples/smartquery.py#L386)
 
+
+> ** Experimental ** 
+> Additional logic (OR, AND, NOT etc) can be expressed using a nested structure for filters, with sqlalchemy operators (or any callable) as keys:
+> ```
+> from sqlalchemy import or_
+> Comment.smart_query(filters={ or_: {
+> 'post___public': True, 
+> 'user__isnull': False
+> }})
+> ```
+> See [this example](examples/smartquery.py#L409) for more details
+
+
 ![icon](http://i.piccy.info/i9/c7168c8821f9e7023e32fd784d0e2f54/1489489664/1113/1127895/rsz_18_256.png)
 See [full example](examples/smartquery.py) and [tests](sqlalchemy_mixins/tests/test_smartquery.py)
 

examples/smartquery.py

@@ -406,6 +406,36 @@ class Comment(BaseModel):
  schema=schema).all()
 log(res) # cm21
 
+##### 3.2.3 Logical operators and arbitrary expressions in filters
+# If we want to use OR, NOT or other logical operators in our queries
+# we can nest the filters dictionary:
+
+res = Post.smart_query(filters={
+ sa.or_: {'archived': True, 'is_commented_by_user': u3}
+})
+log(res) # p11, p22
+
+# Some logic cannot be expressed without using a list instead, e.g.
+# (X OR Y) AND (W OR Z)
+
+# E.g. (somewhat contrived example):
+# (non-archived OR has comments) AND 
+# (user_name like 'B%' or user_name like 'C%')
+res = Post.smart_query(filters=[
+ {sa.or_: {'archived': False, 'comments__isnull': False }},
+ {sa.or_: [
+ {'user___name__like': 'B%'},
+ {'user___name__like': 'C%'}
+ ]}
+])
+
+# !! NOTE !! This cannot be used with the where method, e.g. 
+# Post.where(**{sa.or: {...}})
+# TypeError!! (only strings are allowed as keyword arguments)
+
+# Tested with sa.or_, sa.and_ and sa._not. Other functions that
+# return a sqla expression should also work
+
 ##### 3.3 auto eager load in where() and sort() with auto-joined relations ####
 """
 Smart_query does auto-joins for filtering/sorting,

sqlalchemy_mixins/smartquery.py

@@ -4,7 +4,7 @@
 except ImportError: # pragma: no cover
  pass
 
-from collections import OrderedDict
+from collections import abc, OrderedDict
 
 
 from sqlalchemy import asc, desc, inspect
@@ -24,6 +24,48 @@
 DESC_PREFIX = '-'
 
 
+def _flatten_filter_keys(filters):
+ """
+ :type filters: dict|list
+ Flatten the nested filters, extracting keys where they correspond 
+ to smart_query paths, e.g. 
+ {or_: {'id__gt': 1000, and_ : {
+ 'id__lt': 500,
+ 'related___property__in': (1,2,3) 
+ }}}
+ 
+ Yields:
+
+ 'id__gt', 'id__lt', 'related___property__in'
+
+ Also allow lists (any abc.Sequence subclass) to enable support
+ of expressions like.
+
+ (X OR Y) AND (W OR Z)
+
+ { and_: [
+ {or_: {'id__gt': 5, 'related_id__lt': 10}},
+ {or_: {'related_id2__gt': 1, 'name__like': 'Bob' }}
+ ]}
+ """
+
+ if isinstance(filters, abc.Mapping):
+ for key, value in filters.items():
+ if callable(key):
+ yield from _flatten_filter_keys(value)
+ else:
+ yield key
+
+ elif isinstance(filters, abc.Sequence):
+ for f in filters:
+ yield from _flatten_filter_keys(f)
+
+ else:
+ raise TypeError(
+ "Unsupported type (%s) in filters: %r", (type(filters), filters)
+ )
+
+
 def _parse_path_and_make_aliases(entity, entity_path, attrs, aliases):
  """
  :type entity: InspectionMixin
@@ -51,12 +93,16 @@ def _parse_path_and_make_aliases(entity, entity_path, attrs, aliases):
  relations[relation_name] = [nested_attr]
 
  for relation_name, nested_attrs in relations.items():
- path = entity_path + RELATION_SPLITTER + relation_name \
- if entity_path else relation_name
+ path = (
+ entity_path + RELATION_SPLITTER + relation_name
+ if entity_path
+ else relation_name
+ )
  if relation_name not in entity.relations:
- raise KeyError("Incorrect path `{}`: "
- "{} doesnt have `{}` relationship "
- .format(path, entity, relation_name))
+ raise KeyError(
+ "Incorrect path `{}`: "
+ "{} doesnt have `{}` relationship ".format(path, entity, relation_name)
+ )
  relationship = getattr(entity, relation_name)
  alias = aliased(relationship.property.mapper.class_)
  aliases[path] = alias, relationship
@@ -109,7 +155,7 @@ def smart_query(query, filters=None, sort_attrs=None, schema=None):
  query.session = sess
 
  root_cls = _get_root_cls(query) # for example, User or Post
- attrs = list(filters.keys()) + \
+ attrs = list(_flatten_filter_keys(filters)) + \
  list(map(lambda s: s.lstrip(DESC_PREFIX), sort_attrs))
  aliases = OrderedDict({})
  _parse_path_and_make_aliases(root_cls, '', attrs, aliases)
@@ -122,17 +168,28 @@ def smart_query(query, filters=None, sort_attrs=None, schema=None):
  .options(contains_eager(relationship_path, alias=al[0]))
  loaded_paths.append(relationship_path)
 
- for attr, value in filters.items():
- if RELATION_SPLITTER in attr:
- parts = attr.rsplit(RELATION_SPLITTER, 1)
- entity, attr_name = aliases[parts[0]][0], parts[1]
- else:
- entity, attr_name = root_cls, attr
- try:
- query = query.filter(*entity.filter_expr(**{attr_name: value}))
- except KeyError as e:
- raise KeyError("Incorrect filter path `{}`: {}"
- .format(attr, e))
+ def recurse_filters(_filters):
+ if isinstance(_filters, abc.Mapping):
+ for attr, value in _filters.items():
+ if callable(attr):
+ # E.g. or_, and_, or other sqlalchemy expression
+ yield attr(*recurse_filters(value))
+ continue
+ if RELATION_SPLITTER in attr:
+ parts = attr.rsplit(RELATION_SPLITTER, 1)
+ entity, attr_name = aliases[parts[0]][0], parts[1]
+ else:
+ entity, attr_name = root_cls, attr
+ try:
+ yield from entity.filter_expr(**{attr_name: value})
+ except KeyError as e:
+ raise KeyError("Incorrect filter path `{}`: {}".format(attr, e))
+
+ elif isinstance(_filters, abc.Sequence):
+ for f in _filters:
+ yield from recurse_filters(f)
+
+ query = query.filter(*recurse_filters(filters))
 
  for attr in sort_attrs:
  if RELATION_SPLITTER in attr:

sqlalchemy_mixins/tests/test_smartquery.py

@@ -528,6 +528,62 @@ def test_combinations(self):
  res = Post.where(public=False, is_commented_by_user=u1).all()
  self.assertEqual(set(res), {p11})
 
+ def test_simple_expressions(self):
+ u1, u2, u3, p11, p12, p21, p22, cm11, cm12, cm21, cm22, cm_empty = \
+ self._seed()
+
+ res = Post.smart_query(filters={sa.or_: {'archived': True, 'is_commented_by_user': u3}}).all()
+ self.assertEqual(set(res), {p11, p22})
+
+ def test_nested_expressions(self):
+ u1, u2, u3, p11, p12, p21, p22, cm11, cm12, cm21, cm22, cm_empty = \
+ self._seed()
+
+ # Archived posts, or (has 2016 comment rating != 1)
+ res = Post.smart_query(filters={sa.or_: {
+ 'public': False,
+ sa.and_: {
+ sa.not_: {'comments___rating': 1},
+ 'comments___created_at__year': 2016
+ }
+ }
+ })
+ self.assertEqual(set(res), {p11, p22})
+
+ def test_lists_in_filters_using_explicit_and(self):
+ # Check for users with (post OR comment) AND (name like 'B%' OR id>10)
+ # This cannot be expressed without a list in the filter structure
+ # (would require duplicated or_ keys)
+ u1, u2, u3, p11, p12, p21, p22, cm11, cm12, cm21, cm22, cm_empty = \
+ self._seed()
+
+ res = User.smart_query(filters={
+ sa.and_: [
+ { sa.or_: {
+ 'comments__isnull': False,
+ 'posts__isnull': False
+ }},
+ {sa.or_: {'name__like': 'B%', 'id__gt':10}}
+ ]
+ })
+
+ self.assertEqual(set(res), {u1, u3})
+
+ def test_top_level_list_in_expression(self):
+ # Check for users with (post OR comment) AND (name like 'B%'),
+ # As above, but implicit AND
+ u1, u2, u3, p11, p12, p21, p22, cm11, cm12, cm21, cm22, cm_empty = \
+ self._seed() 
+ res = User.smart_query(filters=[
+ { sa.or_: {
+ 'comments__isnull': False,
+ 'posts__isnull': False
+ }},
+ {sa.or_: {'name__like': 'B%', 'id__gt':10}}
+ ])
+
+ self.assertEqual(set(res), {u1, u3})
+
 
 # noinspection PyUnusedLocal
 class TestSmartQuerySort(BaseTest):

sqlalchemy_mixins/tests/test_timestamp.py

@@ -1,3 +1,4 @@
+import time
 import unittest
 import time
 from datetime import datetime