From ae3dad856cb531956b99fb68ff6cec23c98166ee Mon Sep 17 00:00:00 2001 From: Fred Trotter <83133+ftrotter@users.noreply.github.com> Date: Sun, 17 Jul 2022 17:27:31 -0400 Subject: [PATCH 01/25] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 8d8052d..ea08806 100644 --- a/README.md +++ b/README.md @@ -95,3 +95,5 @@ When your query string is ready to be passed to the function that will execute t SELECT col2, nt.id FROM ex_db.dbo.ex_table tbl LEFT JOIN ex_db.dbo.new_tbl nt ON tbl.id = nt.id AND tbl.city = nt.city WHERE col1 = 1 OR col2 IS NULL ``` NOTE: the SQL constructed is **not** validated. + +Test edit From ea08c254f7b19dd99053df5d736e00935af00dc0 Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Sun, 17 Jul 2022 17:32:56 -0400 Subject: [PATCH 02/25] print function has parens now --- doc_example.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc_example.py b/doc_example.py index 99a729a..9211a0c 100644 --- a/doc_example.py +++ b/doc_example.py @@ -12,13 +12,13 @@ ">>> new_query.s += 'col4' # can take single strings", ">>> new_query.w += 'col1 = 1' # can also take a list", ">>> new_query.w |= 'col2 IS NULL' # handles &= and |= operators", - ">>> print new_query", + ">>> print(new_query)", ">>> new_query", ">>> new_query.s.clear() # clear SELECT component", ">>> new_query.s += 'col1'", ">>> new_query", ">>> new_query.s[0] = 'col2'", - ">>> print new_query.s", + ">>> print(new_query.s)", ">>> new_query.j += 'ex_db.dbo.new_tbl nt ON tbl.id = nt.id'", ">>> new_query.s += 'nt.id'", ">>> new_query", @@ -35,9 +35,9 @@ def main(): for c in commands: - print c + print(c) if c in ('>>> new_query', '>>> new_query.statement'): - print eval(c[4:]) + print(eval(c[4:])) else: exec(c[4:]) From 7240dcfe7ae195db17b4383d36ddf67b831350a9 Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Sun, 17 Jul 2022 17:35:55 -0400 Subject: [PATCH 03/25] manually importing zip --- test/unit_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/unit_tests.py b/test/unit_tests.py index afc36ae..4c39ae4 100644 --- a/test/unit_tests.py +++ b/test/unit_tests.py @@ -1,3 +1,4 @@ +from builtins import zip import unittest as ut from querpy import * @@ -367,4 +368,4 @@ def test_invalid_num_items_passed_as_args(self): if __name__ == '__main__': - ut.main() \ No newline at end of file + ut.main() From 2a17ff63dd2e25401723741a09ed5f30ada28f09 Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Sun, 17 Jul 2022 17:42:47 -0400 Subject: [PATCH 04/25] now its 3 --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ea08806..3d0806d 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,9 @@ The Query class is intended to provide a high level interface for building/editing SQL query strings. +This library should now support python 3. + + Example usage: ```python >>> from querpy import Query @@ -96,4 +99,3 @@ When your query string is ready to be passed to the function that will execute t ``` NOTE: the SQL constructed is **not** validated. -Test edit From 28267978b5c11960832533f84551fee82aa2031a Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Sun, 17 Jul 2022 17:44:23 -0400 Subject: [PATCH 05/25] newline --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3d0806d..268b453 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ building/editing SQL query strings. This library should now support python 3. + Example usage: ```python >>> from querpy import Query From a415ded8d2d08485b8abc796698fdbf87e854100 Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Mon, 18 Jul 2022 00:10:48 -0400 Subject: [PATCH 06/25] documented two regex --- querpy.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/querpy.py b/querpy.py index 2e1cc76..b511ce7 100644 --- a/querpy.py +++ b/querpy.py @@ -23,8 +23,10 @@ class Query(object): - pattern = re.compile('(^\s+|(?<=\s)\s+|\s+$)') - clean_up = re.compile('(?<=WHERE )\s.*?AND|(?<=WHERE )\s.*?OR') + # A series of precompiled regex to perfom various SQL related string tasks + + whitespace_regex = re.compile('(^\s+|(?<=\s)\s+|\s+$)') + where_clean_up = re.compile('(?<=WHERE )\s.*?AND|(?<=WHERE )\s.*?OR') fmt = re.compile('\s(?=FROM)|\s(?=WHERE)|\s(?=GROUP BY)') fmt_after = re.compile( @@ -51,8 +53,8 @@ def __init__(self): @property def statement(self): elements = [self.s(), self.f(), self.j(), self.w(), self.g()] - full_statement = re.subn(self.clean_up, '', ' '.join(elements))[0] - full_statement = re.subn(self.pattern, '', full_statement)[0] + full_statement = re.subn(self.where_clean_up, '', ' '.join(elements))[0] # removes messy contents of WHERE statements? Note sure why this is needed or why it is run on the whole SQL statement + full_statement = re.subn(self.whitespace_regex, '', full_statement)[0] # flattens pretty print SQL to a single line by removing whitespace if full_statement: return full_statement else: @@ -288,4 +290,4 @@ def replace_and(match): string = match.group(0) raw_newlines = re.subn('AND', '\n AND', string)[0] out = re.subn('(?<=BETWEEN)( \w+? )\n\s*?(AND)', r'\1\2', raw_newlines)[0] - return out \ No newline at end of file + return out From ec8a7ed001d37fd531c389f3b2f47244dfd564d0 Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Mon, 18 Jul 2022 00:44:49 -0400 Subject: [PATCH 07/25] learned what the underlying purpose of the query components are and documented them --- querpy.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/querpy.py b/querpy.py index b511ce7..e45ca13 100644 --- a/querpy.py +++ b/querpy.py @@ -25,9 +25,11 @@ class Query(object): # A series of precompiled regex to perfom various SQL related string tasks + # These help with merging all of the statements into a single line whitespace_regex = re.compile('(^\s+|(?<=\s)\s+|\s+$)') where_clean_up = re.compile('(?<=WHERE )\s.*?AND|(?<=WHERE )\s.*?OR') + # These help with the SQL pretty print implementation fmt = re.compile('\s(?=FROM)|\s(?=WHERE)|\s(?=GROUP BY)') fmt_after = re.compile( '(?<=SELECT)\s|(?<=FROM)\s|(?<=WHERE)\s|(?<=GROUP BY)\s' @@ -43,6 +45,9 @@ class Query(object): fmt_and = re.compile('(?<=WHERE).*$', flags=re.S) fmt_or = re.compile('OR') + + + def __init__(self): self.s = SelectComponent() self.f = QueryComponent('FROM') @@ -52,10 +57,12 @@ def __init__(self): @property def statement(self): + # Merges the various SQL componenets into a single SQL statement elements = [self.s(), self.f(), self.j(), self.w(), self.g()] full_statement = re.subn(self.where_clean_up, '', ' '.join(elements))[0] # removes messy contents of WHERE statements? Note sure why this is needed or why it is run on the whole SQL statement full_statement = re.subn(self.whitespace_regex, '', full_statement)[0] # flattens pretty print SQL to a single line by removing whitespace if full_statement: + #Then our regex and merging has worked and return the single line of SQL return full_statement else: return '' @@ -88,7 +95,12 @@ def join_type(self, value): self.j.join_type = value def __str__(self): - query = self.statement + # When we just print the object, we want to assume that we will pretty-print the SQL. + # This section handles the conversion of the single line query, into a pretty printed version.. + # This section could be better implemented using a call to sqlparse + # https://github.com/andialbrecht/sqlparse + # But doing it this way keeps the dependancies low, which is important + query = self.statement # This is the single line query gotten from the statement function query = re.subn(self.fmt, '\n ', query)[0] query = re.subn(self.fmt_after, '\n ', query)[0] query = re.subn(self.fmt_join, '\n ', query)[0] @@ -102,6 +114,9 @@ def __str__(self): class QueryComponent(object): + #This is the base class that everything else will be added to.. + # this is where the magic of += is handled, which makes it easy + # to add things quickly to any componene of the overall query.. def __init__(self, header, sep=''): self.header = header + ' ' @@ -112,7 +127,7 @@ def __iadd__(self, item): self.add_item(item) return self - __iand__ = __ior__ = __iadd__ + __iand__ = __ior__ = __iadd__ # lets set the default for &= and |= to be just += to start.. def add_item(self, item, prefix=''): if prefix: @@ -149,6 +164,8 @@ def __str__(self): class SelectComponent(QueryComponent): + # This models the SELECT component, and sends great energy ensuring that the "DISTINCT" and "TOP" syntax are supported + # otherwise the actual columns are just stored as a list, which is handled by the parent class. header = 'SELECT' dist_pattern = re.compile(' DISTINCT') @@ -204,6 +221,11 @@ def top(self, value): class JoinComponent(QueryComponent): + # like most query components, the joins are just a list of strings... + # The exception is that the type of join is stored as a seperate + # one would think that this allows for retyping the join later.. but really it just means that we + # do not need to add the word "join" to our string storage... so it just handling the fact that the type of join + # is listed before the word 'JOIN' while the method of the join is listed after.. def __init__(self, sep = ''): QueryComponent.__init__(self, '', sep) @@ -227,7 +249,7 @@ def __iadd__(self, item): self.add_item(item, join) return self - __iand__ = __ior__ = __iadd__ + __iand__ = __ior__ = __iadd__ # again, to start, lets have &= and |= just be the same function as += def __call__(self): if self.components: @@ -248,10 +270,11 @@ def __iand__(self, item): return self def __ior__(self, item): + # add this to the list, but with a seperator of 'OR', this will be called when someone uses |= self.add_item(item, 'OR') return self - __iadd__ = __iand__ + __iadd__ = __iand__ # unless we use |= (which will invoke our custom built or function) we are using an "AND" def __str__(self): components = self.components From ed6b25a21b83b06b7fc4455cc67fabb95770763e Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Mon, 18 Jul 2022 00:46:18 -0400 Subject: [PATCH 08/25] documentation lists an add command but then shows the results of an or... fixed --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 268b453..2adb75c 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Example usage: >>> new_query.s += ['col1', 'col2', 'col3'] # can take lists >>> new_query.s += 'col4' # can take single strings >>> new_query.w += 'col1 = 1' # can also take a list (separated by AND) - >>> new_query.w &= 'col2 IS NULL' # handles &= and |= operators + >>> new_query.w |= 'col2 IS NULL' # handles &= and |= operators >>> print new_query SELECT col1, From 1300fcde4699537d0ec7ec524008cd3a64c651aa Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Mon, 18 Jul 2022 01:03:39 -0400 Subject: [PATCH 09/25] remove those tabs --- querpy.py | 77 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/querpy.py b/querpy.py index e45ca13..3b79676 100644 --- a/querpy.py +++ b/querpy.py @@ -57,12 +57,12 @@ def __init__(self): @property def statement(self): - # Merges the various SQL componenets into a single SQL statement + # Merges the various SQL componenets into a single SQL statement elements = [self.s(), self.f(), self.j(), self.w(), self.g()] full_statement = re.subn(self.where_clean_up, '', ' '.join(elements))[0] # removes messy contents of WHERE statements? Note sure why this is needed or why it is run on the whole SQL statement full_statement = re.subn(self.whitespace_regex, '', full_statement)[0] # flattens pretty print SQL to a single line by removing whitespace if full_statement: - #Then our regex and merging has worked and return the single line of SQL + #Then our regex and merging has worked and return the single line of SQL return full_statement else: return '' @@ -95,17 +95,17 @@ def join_type(self, value): self.j.join_type = value def __str__(self): - # When we just print the object, we want to assume that we will pretty-print the SQL. - # This section handles the conversion of the single line query, into a pretty printed version.. - # This section could be better implemented using a call to sqlparse - # https://github.com/andialbrecht/sqlparse - # But doing it this way keeps the dependancies low, which is important + # When we just print the object, we want to assume that we will pretty-print the SQL. + # This section handles the conversion of the single line query, into a pretty printed version.. + # This section could be better implemented using a call to sqlpars + # https://github.com/andialbrecht/sqlparse + # But doing it this way keeps the dependancies low, which is important query = self.statement # This is the single line query gotten from the statement function query = re.subn(self.fmt, '\n ', query)[0] query = re.subn(self.fmt_after, '\n ', query)[0] query = re.subn(self.fmt_join, '\n ', query)[0] query = re.subn(self.fmt_commas, '\n ', query)[0] - query = re.subn(self.fmt_and, replace_and, query)[0] + query = re.subn(self.fmt_and, Query.replace_and, query)[0] query = re.subn(self.fmt_or, '\n OR', query)[0] return query @@ -113,6 +113,41 @@ def __str__(self): __repr__ = __str__ + @staticmethod + def build_join(*args): + # A static helper function to build a join + # this assumes that the first argument is the table... + # and that every subsequent pair of arguments is something to join 'ON' + + tbl_name = args[0] + args = args[1:] + if len(args) % 2 != 0 or args == (): + raise BaseException( + 'You must provide an even number of columns to join on.' + ) + + args_expr = ['{0} = {1}'.format(args[2 * i], args[2 * i + 1]) + for i in range(int(len(args) / 2))] # int() for Python 3 + args_expr = ' AND '.join(args_expr) + join_str = ' '.join([tbl_name, 'ON', args_expr]) + + return join_str + + @staticmethod + def replace_and(match): + """ + helper function for indenting AND in WHERE clause + """ + string = match.group(0) + raw_newlines = re.subn('AND', '\n AND', string)[0] + out = re.subn('(?<=BETWEEN)( \w+? )\n\s*?(AND)', r'\1\2', raw_newlines)[0] + return out + + + + + + class QueryComponent(object): #This is the base class that everything else will be added to.. # this is where the magic of += is handled, which makes it easy @@ -288,29 +323,3 @@ def __str__(self): __repr__ = __str__ - - -def build_join(*args): - tbl_name = args[0] - args = args[1:] - if len(args) % 2 != 0 or args == (): - raise BaseException( - 'You must provide an even number of columns to join on.' - ) - - args_expr = ['{0} = {1}'.format(args[2 * i], args[2 * i + 1]) - for i in range(int(len(args) / 2))] # int() for Python 3 - args_expr = ' AND '.join(args_expr) - join_str = ' '.join([tbl_name, 'ON', args_expr]) - - return join_str - - -def replace_and(match): - """ - helper function for indenting AND in WHERE clause - """ - string = match.group(0) - raw_newlines = re.subn('AND', '\n AND', string)[0] - out = re.subn('(?<=BETWEEN)( \w+? )\n\s*?(AND)', r'\1\2', raw_newlines)[0] - return out From 190c9d98b77012a22ed0e2682a4e76fd345430bd Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Mon, 18 Jul 2022 01:05:58 -0400 Subject: [PATCH 10/25] correct build_join example in README --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2adb75c..52a219e 100644 --- a/README.md +++ b/README.md @@ -75,12 +75,11 @@ Suppose you want to extend your query by joining to another table and adding col col1 = 1 OR col2 IS NULL ``` -While this works, we are returning to the land of long strings. We can do the same thing (n.b. we'll LEFT JOIN this time) using the build_join helper function to make the join step more readable and modular: +While this works, we are returning to the land of long strings. We can do the same thing (n.b. we'll LEFT JOIN this time) using the Query.build_join helper function to make the join step more readable and modular: ```python - >>> from querpy import build_join >>> new_query.j.clear() >>> new_query.join_type = 'LEFT' - >>> new_query.j += build_join('ex_db.dbo.new_tbl nt', 'tbl.id', 'nt.id', 'tbl.city', 'nt.city') + >>> new_query.j += Query.build_join('ex_db.dbo.new_tbl nt', 'tbl.id', 'nt.id', 'tbl.city', 'nt.city') >>> new_query.join_type = '' # set back to regular join >>> new_query SELECT From 71e6239a73e329fc0e31f798c3603e3bcde9a04c Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Mon, 18 Jul 2022 01:06:55 -0400 Subject: [PATCH 11/25] remove build_join as a seperate function --- doc_example.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc_example.py b/doc_example.py index 9211a0c..cf79ac9 100644 --- a/doc_example.py +++ b/doc_example.py @@ -22,10 +22,9 @@ ">>> new_query.j += 'ex_db.dbo.new_tbl nt ON tbl.id = nt.id'", ">>> new_query.s += 'nt.id'", ">>> new_query", - ">>> from querpy import build_join", ">>> new_query.j.clear()", ">>> new_query.join_type = 'LEFT'", - ">>> new_query.j += build_join('ex_db.dbo.new_tbl nt', 'tbl.id', 'nt.id'," + ">>> new_query.j += Query.build_join('ex_db.dbo.new_tbl nt', 'tbl.id', 'nt.id'," " 'tbl.city', 'nt.city')", ">>> new_query.join_type = '' # set back to regular join", ">>> new_query", From f77c4615915dbf0e51c67950e01efd001e9f51fc Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Mon, 18 Jul 2022 01:08:49 -0400 Subject: [PATCH 12/25] account for static functions in test --- test/unit_tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/unit_tests.py b/test/unit_tests.py index 4c39ae4..4da6c4d 100644 --- a/test/unit_tests.py +++ b/test/unit_tests.py @@ -288,7 +288,7 @@ def test_fmt_and(self): 'JOIN tbl2 ON col1 = col2 AND col3 = col4 ' 'WHERE col1 = col3 AND col4 = col5 AND col5 = col6' ) - subbed = re.subn(self.query.fmt_and, replace_and, string)[0] + subbed = re.subn(self.query.fmt_and, Query.replace_and, string)[0] self.assertEqual( subbed, 'JOIN tbl2 ON col1 = col2 AND col3 = col4 ' @@ -335,7 +335,7 @@ def test_top(self): def test_print(self): self.query.s += ['col1', 'col2', 'col3'] self.query.f += 'tbl1 t1' - self.query.j += build_join('tbl2 t2', 't1.id', 't2.id', 't1.city', + self.query.j += Query.build_join('tbl2 t2', 't1.id', 't2.id', 't1.city', 't2.city') self.query.w += ['col1 IS NULL', 'col4 BETWEEN col1 AND col2', 'col2 = t1.id', 'col3 BETWEEN 0 AND 10'] @@ -356,14 +356,14 @@ def setUp(self): self.item2 = ['tbl2 t2', 't2.id', 'oid', 't2.city', 'city'] def test_join_valid_items(self): - to_test1 = build_join(*self.item1) - to_test2 = build_join(*self.item2) + to_test1 = Query.build_join(*self.item1) + to_test2 = Query.build_join(*self.item2) self.assertEqual(to_test1, 'tbl1 t1 ON t1.id = oid') self.assertEqual(to_test2, 'tbl2 t2 ON t2.id = oid AND t2.city = city') def test_invalid_num_items_passed_as_args(self): invalid = self.item2[:-1] - self.assertRaises(BaseException, build_join, invalid) + self.assertRaises(BaseException, Query.build_join, invalid) if __name__ == '__main__': From 9f6929506a22efa09751f0eb4993f09044777d24 Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Mon, 18 Jul 2022 01:09:56 -0400 Subject: [PATCH 13/25] use build_join as static function --- doc_example.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc_example.txt b/doc_example.txt index 4b31073..313a283 100644 --- a/doc_example.txt +++ b/doc_example.txt @@ -53,10 +53,9 @@ SELECT WHERE col1 = 1 OR col2 IS NULL ->>> from querpy import build_join >>> new_query.j.clear() >>> new_query.join_type = 'LEFT' ->>> new_query.j += build_join('ex_db.dbo.new_tbl nt', 'tbl.id', 'nt.id', 'tbl.city', 'nt.city') +>>> new_query.j += Query.build_join('ex_db.dbo.new_tbl nt', 'tbl.id', 'nt.id', 'tbl.city', 'nt.city') >>> new_query.join_type = '' # set back to regular join >>> new_query SELECT From 8fffa422661f77b3c7388e8d184573c43d7bb521 Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Mon, 18 Jul 2022 02:32:10 -0400 Subject: [PATCH 14/25] added create table function --- doc_example.py | 1 + querpy.py | 46 +++++++++++++++++++++++++++++++++++++--------- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/doc_example.py b/doc_example.py index cf79ac9..0b07cf9 100644 --- a/doc_example.py +++ b/doc_example.py @@ -27,6 +27,7 @@ ">>> new_query.j += Query.build_join('ex_db.dbo.new_tbl nt', 'tbl.id', 'nt.id'," " 'tbl.city', 'nt.city')", ">>> new_query.join_type = '' # set back to regular join", + ">>> new_query.ci += 'thisDB.thatTable' # set back to regular join", ">>> new_query", ">>> new_query.statement", ] diff --git a/querpy.py b/querpy.py index 3b79676..7c509aa 100644 --- a/querpy.py +++ b/querpy.py @@ -13,9 +13,9 @@ >>> new_query # should print full query """ -__author__ = 'Paul Garaud' -__version__ = '0.1' -__date__ = '2015-03-19' +__author__ = 'Paul Garaud, Fred Trotter' +__version__ = '0.2' +__date__ = '2022-06-18' import re @@ -27,9 +27,11 @@ class Query(object): # These help with merging all of the statements into a single line whitespace_regex = re.compile('(^\s+|(?<=\s)\s+|\s+$)') - where_clean_up = re.compile('(?<=WHERE )\s.*?AND|(?<=WHERE )\s.*?OR') + #the design of the where list is such that every element of the list has an 'AND' or 'OR' as a prefix.. + #but the first one after the where does not need that.. so we just look for a WHERE OR or a WHERE AND and remove the 'OR' or 'AND' + where_clean_up = re.compile('(?<=WHERE )\s.*?AND|(?<=WHERE )\s.*?OR') - # These help with the SQL pretty print implementation + # All of these fmt_(something) help with the SQL pretty print implementation fmt = re.compile('\s(?=FROM)|\s(?=WHERE)|\s(?=GROUP BY)') fmt_after = re.compile( '(?<=SELECT)\s|(?<=FROM)\s|(?<=WHERE)\s|(?<=GROUP BY)\s' @@ -49,6 +51,7 @@ class Query(object): def __init__(self): + self.ci = CreateInsertComponent() self.s = SelectComponent() self.f = QueryComponent('FROM') self.j = JoinComponent() @@ -58,7 +61,8 @@ def __init__(self): @property def statement(self): # Merges the various SQL componenets into a single SQL statement - elements = [self.s(), self.f(), self.j(), self.w(), self.g()] + print(self.ci()) + elements = [self.ci(), self.s(), self.f(), self.j(), self.w(), self.g()] full_statement = re.subn(self.where_clean_up, '', ' '.join(elements))[0] # removes messy contents of WHERE statements? Note sure why this is needed or why it is run on the whole SQL statement full_statement = re.subn(self.whitespace_regex, '', full_statement)[0] # flattens pretty print SQL to a single line by removing whitespace if full_statement: @@ -100,7 +104,7 @@ def __str__(self): # This section could be better implemented using a call to sqlpars # https://github.com/andialbrecht/sqlparse # But doing it this way keeps the dependancies low, which is important - query = self.statement # This is the single line query gotten from the statement function + query = self.statement # This is the single line query gotten from the statement function query = re.subn(self.fmt, '\n ', query)[0] query = re.subn(self.fmt_after, '\n ', query)[0] query = re.subn(self.fmt_join, '\n ', query)[0] @@ -146,8 +150,6 @@ def replace_and(match): - - class QueryComponent(object): #This is the base class that everything else will be added to.. # this is where the magic of += is handled, which makes it easy @@ -179,6 +181,8 @@ def clear(self): self.components = list() def __call__(self): + # This is the function that converts the list of items in the querycomponent into a long string + # it is always prefixed by the header.. if self.components: return self.header + self.sep.join(self.components) return '' @@ -197,6 +201,30 @@ def __str__(self): __repr__ = __str__ +class CreateInsertComponent(QueryComponent): + # Implements the very first part of a CREATE TABLE db.table AS or INSERT INTO db.table + # depending on whether the is_first_data_add setting has been set + + + is_first_data_add = True + + def __iadd__(self, item): + #we only have the one item.. + self.components = list() # overwrites whatever was there + if self.is_first_data_add: + #Then this is a CREATE TABLE AS clause + self.header = 'CREATE TABLE ' + item + " AS \n" + else: + self.header = 'INSERT INTO ' + item + " \n" + return self + + def __init__(self): + self.header = '' # by default, this is not used. + self.components = list() + + def __call__(self): + return self.header + class SelectComponent(QueryComponent): # This models the SELECT component, and sends great energy ensuring that the "DISTINCT" and "TOP" syntax are supported From 90e1a8dd7780c0c5f3e344639194fc90809defe6 Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Mon, 18 Jul 2022 13:18:20 -0400 Subject: [PATCH 15/25] remove additional print --- querpy.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/querpy.py b/querpy.py index 7c509aa..678a912 100644 --- a/querpy.py +++ b/querpy.py @@ -57,11 +57,12 @@ def __init__(self): self.j = JoinComponent() self.w = WhereComponent() self.g = QueryComponent('GROUP BY', sep=',') + self.l = LimitComponent() + @property def statement(self): # Merges the various SQL componenets into a single SQL statement - print(self.ci()) elements = [self.ci(), self.s(), self.f(), self.j(), self.w(), self.g()] full_statement = re.subn(self.where_clean_up, '', ' '.join(elements))[0] # removes messy contents of WHERE statements? Note sure why this is needed or why it is run on the whole SQL statement full_statement = re.subn(self.whitespace_regex, '', full_statement)[0] # flattens pretty print SQL to a single line by removing whitespace @@ -226,6 +227,21 @@ def __call__(self): return self.header +class LimitComponent(QueryComponent): + # Adds support for the limit command + + def __iadd__(self, item): + #we only have the one item.. should be like '10, 100' + #we should add a test to make sure this is correct. + self.components = list() # overwrites whatever was there + self.components.append(item) + self.header = ' LIMIT ' + + def __init__(self): + self.header = '' # by default, this is not used. + self.components = list() # none to start + + class SelectComponent(QueryComponent): # This models the SELECT component, and sends great energy ensuring that the "DISTINCT" and "TOP" syntax are supported # otherwise the actual columns are just stored as a list, which is handled by the parent class. From cde45b88e5b16bae749c275cb2e4b41df1e8910a Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Mon, 18 Jul 2022 13:23:57 -0400 Subject: [PATCH 16/25] adding group by to test --- doc_example.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc_example.py b/doc_example.py index 0b07cf9..b6c36fe 100644 --- a/doc_example.py +++ b/doc_example.py @@ -28,6 +28,9 @@ " 'tbl.city', 'nt.city')", ">>> new_query.join_type = '' # set back to regular join", ">>> new_query.ci += 'thisDB.thatTable' # set back to regular join", + ">>> new_query.l += ' 10, 100' ", + ">>> new_query.g += 'col1' ", + ">>> new_query.g += 'col3' ", ">>> new_query", ">>> new_query.statement", ] From 6a140cd9fb738e2f3f34a5e03c8448acbd7daa72 Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Mon, 18 Jul 2022 13:30:16 -0400 Subject: [PATCH 17/25] pretty print is wonky, but LIMIT works --- doc_example.py | 2 +- querpy.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/doc_example.py b/doc_example.py index b6c36fe..57a1eef 100644 --- a/doc_example.py +++ b/doc_example.py @@ -28,7 +28,7 @@ " 'tbl.city', 'nt.city')", ">>> new_query.join_type = '' # set back to regular join", ">>> new_query.ci += 'thisDB.thatTable' # set back to regular join", - ">>> new_query.l += ' 10, 100' ", + ">>> new_query.li += ' 10, 100' ", ">>> new_query.g += 'col1' ", ">>> new_query.g += 'col3' ", ">>> new_query", diff --git a/querpy.py b/querpy.py index 678a912..55dbaa7 100644 --- a/querpy.py +++ b/querpy.py @@ -57,13 +57,13 @@ def __init__(self): self.j = JoinComponent() self.w = WhereComponent() self.g = QueryComponent('GROUP BY', sep=',') - self.l = LimitComponent() + self.li = LimitComponent() @property def statement(self): # Merges the various SQL componenets into a single SQL statement - elements = [self.ci(), self.s(), self.f(), self.j(), self.w(), self.g()] + elements = [self.ci(), self.s(), self.f(), self.j(), self.w(), self.g(), self.li()] full_statement = re.subn(self.where_clean_up, '', ' '.join(elements))[0] # removes messy contents of WHERE statements? Note sure why this is needed or why it is run on the whole SQL statement full_statement = re.subn(self.whitespace_regex, '', full_statement)[0] # flattens pretty print SQL to a single line by removing whitespace if full_statement: @@ -235,7 +235,9 @@ def __iadd__(self, item): #we should add a test to make sure this is correct. self.components = list() # overwrites whatever was there self.components.append(item) - self.header = ' LIMIT ' + self.header = " LIMIT " + self.sep = '' + return self def __init__(self): self.header = '' # by default, this is not used. From 5a550a1cfdb87ce38ae5f685919b5cd226809ae6 Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Mon, 18 Jul 2022 13:31:45 -0400 Subject: [PATCH 18/25] just use l for LIMIT --- doc_example.py | 2 +- querpy.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc_example.py b/doc_example.py index 57a1eef..b6c36fe 100644 --- a/doc_example.py +++ b/doc_example.py @@ -28,7 +28,7 @@ " 'tbl.city', 'nt.city')", ">>> new_query.join_type = '' # set back to regular join", ">>> new_query.ci += 'thisDB.thatTable' # set back to regular join", - ">>> new_query.li += ' 10, 100' ", + ">>> new_query.l += ' 10, 100' ", ">>> new_query.g += 'col1' ", ">>> new_query.g += 'col3' ", ">>> new_query", diff --git a/querpy.py b/querpy.py index 55dbaa7..a64d479 100644 --- a/querpy.py +++ b/querpy.py @@ -57,13 +57,13 @@ def __init__(self): self.j = JoinComponent() self.w = WhereComponent() self.g = QueryComponent('GROUP BY', sep=',') - self.li = LimitComponent() + self.l = LimitComponent() @property def statement(self): # Merges the various SQL componenets into a single SQL statement - elements = [self.ci(), self.s(), self.f(), self.j(), self.w(), self.g(), self.li()] + elements = [self.ci(), self.s(), self.f(), self.j(), self.w(), self.g(), self.l()] full_statement = re.subn(self.where_clean_up, '', ' '.join(elements))[0] # removes messy contents of WHERE statements? Note sure why this is needed or why it is run on the whole SQL statement full_statement = re.subn(self.whitespace_regex, '', full_statement)[0] # flattens pretty print SQL to a single line by removing whitespace if full_statement: From daefae3d811071b0c6dac1873568eb0c787de120 Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Mon, 18 Jul 2022 13:35:46 -0400 Subject: [PATCH 19/25] create table, group by and limit examples in the README --- README.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/README.md b/README.md index 52a219e..2337e8d 100644 --- a/README.md +++ b/README.md @@ -97,5 +97,33 @@ When your query string is ready to be passed to the function that will execute t >>> new_query.statement SELECT col2, nt.id FROM ex_db.dbo.ex_table tbl LEFT JOIN ex_db.dbo.new_tbl nt ON tbl.id = nt.id AND tbl.city = nt.city WHERE col1 = 1 OR col2 IS NULL ``` + +You can also prepend a CREATE TABLE <> AS to the SQL +```python + >>> new_query.j += 'ex_db.dbo.new_tbl nt ON tbl.id = nt.id' + >>> new_query.s += 'nt.id' + >>> new_query.g += 'col1' + >>> new_query.l += ' 10, 100' + >>> new_query.ci += ' db_name.tbl_name ' + >>> new_query + CREATE TABLE db_name.tbl_name AS + SELECT + col1, + nt.id + FROM + ex_db.dbo.ex_table tbl + JOIN ex_db.dbo.new_tbl nt ON tbl.id = nt.id + WHERE + col1 = 1 + OR col2 IS NULL + GROUP BY + col1 + LIMIT 10, 100 +``` + + +Suppose you want to extend your query by joining to another table and adding columns from this table: + + NOTE: the SQL constructed is **not** validated. From 6b11d7e62a2a67894efc280c7588bc531eb8b516 Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Thu, 4 Aug 2022 02:52:34 -0400 Subject: [PATCH 20/25] adding order by and notes --- querpy.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/querpy.py b/querpy.py index a64d479..8acf449 100644 --- a/querpy.py +++ b/querpy.py @@ -57,13 +57,14 @@ def __init__(self): self.j = JoinComponent() self.w = WhereComponent() self.g = QueryComponent('GROUP BY', sep=',') + self.o = QueryComponent('ORDER BY', sep=',') self.l = LimitComponent() @property def statement(self): # Merges the various SQL componenets into a single SQL statement - elements = [self.ci(), self.s(), self.f(), self.j(), self.w(), self.g(), self.l()] + elements = [self.ci(), self.s(), self.f(), self.j(), self.w(), self.g(), self.o(), self.l()] full_statement = re.subn(self.where_clean_up, '', ' '.join(elements))[0] # removes messy contents of WHERE statements? Note sure why this is needed or why it is run on the whole SQL statement full_statement = re.subn(self.whitespace_regex, '', full_statement)[0] # flattens pretty print SQL to a single line by removing whitespace if full_statement: @@ -72,6 +73,17 @@ def statement(self): else: return '' + #Some properties that allow us to expose some of the variables in the QueryComponents, as thought they were + #directly on the main Query object + + @property + def is_first_data_add(self): + return self.ci.is_first_data_add + + @is_first_data_add.setter + def is_first_data_add(self, value): + self.ci.is_first_data_add = value + @property def distinct(self): return self.s.distinct @@ -205,7 +217,10 @@ def __str__(self): class CreateInsertComponent(QueryComponent): # Implements the very first part of a CREATE TABLE db.table AS or INSERT INTO db.table # depending on whether the is_first_data_add setting has been set - + # TODO the way this works now, you have to set is_first_data_add BEFORE adding the table + # But it should not care whether it was added first or second.. the is_first_data_add + # Should swap between CREATE TABLE and INSERT INTO whenever it is set.. + # Which means that the behavior of setting the header should happen when the data is being read.. not when it is written... is_first_data_add = True From 3341a65a22648ff696877d30d153019f2bcf918b Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Thu, 4 Aug 2022 02:55:55 -0400 Subject: [PATCH 21/25] testing order by --- doc_example.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc_example.py b/doc_example.py index b6c36fe..fab77c6 100644 --- a/doc_example.py +++ b/doc_example.py @@ -31,6 +31,9 @@ ">>> new_query.l += ' 10, 100' ", ">>> new_query.g += 'col1' ", ">>> new_query.g += 'col3' ", + ">>> new_query.o += 'col1' ", + ">>> new_query.o += 'col3' ", + ">>> new_query", ">>> new_query", ">>> new_query.statement", ] From 7099dfc4b4f4df7037951ad0feda5cfee6d9bb13 Mon Sep 17 00:00:00 2001 From: Fred Trotter <83133+ftrotter@users.noreply.github.com> Date: Wed, 8 Feb 2023 18:34:50 -0500 Subject: [PATCH 22/25] porting Shawns changes back! --- querpy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/querpy.py b/querpy.py index 8acf449..05878de 100644 --- a/querpy.py +++ b/querpy.py @@ -45,7 +45,7 @@ class Query(object): ) fmt_commas = re.compile('(?<=,)\s') fmt_and = re.compile('(?<=WHERE).*$', flags=re.S) - fmt_or = re.compile('OR') + fmt_or = re.compile('\sOR\s') From cc42da099c8f6477ce1e38f96387587a5efa1284 Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Tue, 18 Apr 2023 22:59:54 -0400 Subject: [PATCH 23/25] Thanks to @dmahone1 for finding this bug and providing the initial fix --- querpy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/querpy.py b/querpy.py index 8acf449..2f04bcf 100644 --- a/querpy.py +++ b/querpy.py @@ -45,7 +45,7 @@ class Query(object): ) fmt_commas = re.compile('(?<=,)\s') fmt_and = re.compile('(?<=WHERE).*$', flags=re.S) - fmt_or = re.compile('OR') + fmt_or = re.compile('\sOR\s') @@ -123,7 +123,7 @@ def __str__(self): query = re.subn(self.fmt_join, '\n ', query)[0] query = re.subn(self.fmt_commas, '\n ', query)[0] query = re.subn(self.fmt_and, Query.replace_and, query)[0] - query = re.subn(self.fmt_or, '\n OR', query)[0] + query = re.subn(self.fmt_or, '\n OR ', query)[0] return query From e0b21c6f2d57309ec01e839e91864f0954801cab Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Sat, 23 Nov 2024 00:09:53 -0500 Subject: [PATCH 24/25] handle objects that know how to become strings --- querpy.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/querpy.py b/querpy.py index 2f04bcf..05481f9 100644 --- a/querpy.py +++ b/querpy.py @@ -179,16 +179,25 @@ def __iadd__(self, item): __iand__ = __ior__ = __iadd__ # lets set the default for &= and |= to be just += to start.. + + # Note this is a modification of the original code, which did not have the ability to handle an object + # That knows how to become a string.. like DBTable.... + # TODO write a test that this works correctly for DBTable objects. def add_item(self, item, prefix=''): if prefix: prefix = prefix + ' ' - if type(item) == str: + if isinstance(item, str): # Handle strings self.components.append(''.join([prefix, item])) - elif type(item) == list: + elif isinstance(item, list): # Handle lists items = [''.join([prefix, i]) for i in item] self.components.extend(items) - else: - raise ValueError('Item must be a string or list') + else: # Handle objects by converting them to strings + try: + item_as_string = str(item) + self.components.append(''.join([prefix, item_as_string])) + except Exception: + raise ValueError('Item must be a string, list, or object convertible to string') + def clear(self): self.components = list() From ede04ee71321c3cbcaff075a4c24ce257f52692b Mon Sep 17 00:00:00 2001 From: Fred Trotter Date: Sat, 23 Nov 2024 01:02:36 -0500 Subject: [PATCH 25/25] adding recent changes --- querpy.py | 60 +++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/querpy.py b/querpy.py index 05481f9..e5673ed 100644 --- a/querpy.py +++ b/querpy.py @@ -1,6 +1,8 @@ """ +querpy.py + The Query class is intended to provide a high level interface for -building/editing SQL queries. +building/editing SQL queries. Built before we understood what SQLAlchemy was. Example usage: >>> new_query = Query() @@ -30,6 +32,7 @@ class Query(object): #the design of the where list is such that every element of the list has an 'AND' or 'OR' as a prefix.. #but the first one after the where does not need that.. so we just look for a WHERE OR or a WHERE AND and remove the 'OR' or 'AND' where_clean_up = re.compile('(?<=WHERE )\s.*?AND|(?<=WHERE )\s.*?OR') + having_clean_up = re.compile('(?<=HAVING )\s.*?AND|(?<=HAVING )\s.*?OR') # All of these fmt_(something) help with the SQL pretty print implementation fmt = re.compile('\s(?=FROM)|\s(?=WHERE)|\s(?=GROUP BY)') @@ -44,7 +47,8 @@ class Query(object): ) ) fmt_commas = re.compile('(?<=,)\s') - fmt_and = re.compile('(?<=WHERE).*$', flags=re.S) + fmt_where_and = re.compile('(?<=WHERE).*$', flags=re.S) + fmt_having_and = re.compile('(?<=HAVING).*$', flags=re.S) fmt_or = re.compile('\sOR\s') @@ -57,16 +61,20 @@ def __init__(self): self.j = JoinComponent() self.w = WhereComponent() self.g = QueryComponent('GROUP BY', sep=',') + self.h = HavingComponent() self.o = QueryComponent('ORDER BY', sep=',') self.l = LimitComponent() @property def statement(self): + + where_statement = re.subn(self.where_clean_up, '', self.w())[0] + having_statement = re.subn(self.having_clean_up, '', self.h())[0] + # Merges the various SQL componenets into a single SQL statement - elements = [self.ci(), self.s(), self.f(), self.j(), self.w(), self.g(), self.o(), self.l()] - full_statement = re.subn(self.where_clean_up, '', ' '.join(elements))[0] # removes messy contents of WHERE statements? Note sure why this is needed or why it is run on the whole SQL statement - full_statement = re.subn(self.whitespace_regex, '', full_statement)[0] # flattens pretty print SQL to a single line by removing whitespace + elements = [self.ci(), self.s(), self.f(), self.j(), where_statement, self.g(), having_statement, self.o(), self.l()] + full_statement = re.subn(self.whitespace_regex, '', ' '.join(elements))[0] # flattens pretty print SQL to a single line by removing whitespace if full_statement: #Then our regex and merging has worked and return the single line of SQL return full_statement @@ -122,7 +130,8 @@ def __str__(self): query = re.subn(self.fmt_after, '\n ', query)[0] query = re.subn(self.fmt_join, '\n ', query)[0] query = re.subn(self.fmt_commas, '\n ', query)[0] - query = re.subn(self.fmt_and, Query.replace_and, query)[0] + query = re.subn(self.fmt_where_and, Query.replace_and, query)[0] + query = re.subn(self.fmt_having_and, Query.replace_and, query)[0] query = re.subn(self.fmt_or, '\n OR ', query)[0] return query @@ -179,7 +188,6 @@ def __iadd__(self, item): __iand__ = __ior__ = __iadd__ # lets set the default for &= and |= to be just += to start.. - # Note this is a modification of the original code, which did not have the ability to handle an object # That knows how to become a string.. like DBTable.... # TODO write a test that this works correctly for DBTable objects. @@ -198,7 +206,6 @@ def add_item(self, item, prefix=''): except Exception: raise ValueError('Item must be a string, list, or object convertible to string') - def clear(self): self.components = list() @@ -206,7 +213,8 @@ def __call__(self): # This is the function that converts the list of items in the querycomponent into a long string # it is always prefixed by the header.. if self.components: - return self.header + self.sep.join(self.components) + return_me = self.header + self.sep.join(self.components) + return return_me return '' def __getitem__(self, key): @@ -238,9 +246,9 @@ def __iadd__(self, item): self.components = list() # overwrites whatever was there if self.is_first_data_add: #Then this is a CREATE TABLE AS clause - self.header = 'CREATE TABLE ' + item + " AS \n" + self.header = 'CREATE TABLE ' + str(item) + " AS \n" else: - self.header = 'INSERT INTO ' + item + " \n" + self.header = 'INSERT INTO ' + str(item) + " \n" return self def __init__(self): @@ -393,3 +401,33 @@ def __str__(self): __repr__ = __str__ +class HavingComponent(QueryComponent): + + header = "HAVING" + + def __init__(self, sep=''): + self.header = self.header + ' ' + QueryComponent.__init__(self, self.header, sep) + + def __iand__(self, item): + self.add_item(item, 'AND') + return self + + def __ior__(self, item): + # add this to the list, but with a seperator of 'OR', this will be called when someone uses |= + self.add_item(item, 'OR') + return self + + __iadd__ = __iand__ # unless we use |= (which will invoke our custom built or function) we are using an "AND" + + def __str__(self): + components = self.components + if components: + components = self.components[:] + components[0] = re.sub('^AND |^OR ', '', components[0]) + to_print = list() + for n, c in enumerate(components): + to_print.append("{0}: '{1}'".format(n, c)) + return 'index: item\n' + ', '.join(to_print) + + __repr__ = __str__