From 8aabf0e022d6db13646fcaa023355d4cbc4ba442 Mon Sep 17 00:00:00 2001 From: shinichi-takii Date: Mon, 6 Jul 2020 13:01:49 +0900 Subject: [PATCH 1/3] Fix parse for column-constraint --- .github/PULL_REQUEST_TEMPLATE.md | 23 ++++++++++++++++++- .gitignore | 3 +++ .travis.yml | 2 +- CHANGELOG.md | 13 +++++++++++ README.md | 2 +- ddlparse/ddlparse.py | 39 +++++++++++++++++++++++++++++++- setup.py | 2 +- test/test_ddlparse.py | 32 ++++++++++++++++++-------- tox.ini | 2 +- 9 files changed, 102 insertions(+), 16 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c8950ec..7ff2ae6 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,2 +1,23 @@ -# Changes +## Summary + + +### Added +- {{ added-summary }} + +### Changed +- {{ changed-summary }} + +### Removed +- {{ removed-summary }} + +### Fixed +- {{ fixed-summary }} + + +## File Details +### {{ file-name }} +- {{ summary }} + + +## Applicable Issues - fix # : summary... diff --git a/.gitignore b/.gitignore index a79ce6e..0b73a1e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ .DS_Store *~ +# IDE +.vscode + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/.travis.yml b/.travis.yml index 3234547..efe2382 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,10 +3,10 @@ dist: xenial sudo: true python: - - "3.4" - "3.5" - "3.6" - "3.7" + - "3.8" # command to install dependencies install: diff --git a/CHANGELOG.md b/CHANGELOG.md index c9c4098..1db04a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.5.0] - 2020-07-06 +### Added +- Add supports for Python 3.8 + +### Removed +- End of support for Python 3.4 + +### Fixed +- Fix parse for column-constraint. +- Miner fix. + + ## [1.4.0] - 2019-12-08 ### Added - Add supports to BigQuery `NUMERIC` data type. @@ -130,6 +142,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Initial released. +[1.5.0]: https://github.com/shinichi-takii/ddlparse/compare/v1.4.0...v1.5.0 [1.4.0]: https://github.com/shinichi-takii/ddlparse/compare/v1.3.1...v1.4.0 [1.3.1]: https://github.com/shinichi-takii/ddlparse/compare/v1.3.0...v1.3.1 [1.3.0]: https://github.com/shinichi-takii/ddlparse/compare/v1.2.3...v1.3.0 diff --git a/README.md b/README.md index 39ccde1..a4d3f5c 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ ## Requirement -1. Python >= 3.4 +1. Python >= 3.5 1. [pyparsing](https://github.com/pyparsing/pyparsing) ## Installation diff --git a/ddlparse/ddlparse.py b/ddlparse/ddlparse.py index 24a7659..ede6c32 100644 --- a/ddlparse/ddlparse.py +++ b/ddlparse/ddlparse.py @@ -493,6 +493,43 @@ class DdlParse(DdlParseBase): _COMMENT = Suppress("--" + Regex(r".+")) + _COLUMN_CONSTRAINT_BASE = r""" + (?!--) + ( + ( + \s*\b(?:NOT\s+)NULL?\b + )? + ( + \s*\bAUTO_INCREMENT\b + )?( + \s*\b(UNIQUE|PRIMARY)(?:\s+KEY)?\b + )? + ( + \s*\bDEFAULT\b\s+( + ([A-Za-z0-9_\.\'\" -]|[^\x01-\x7E])*\:\:[A-Za-z0-9\[\]]+ + | + \'(\\\'|[^\']|,)+\' + | + \"(\\\"|[^\"]|,)+\" + | + [^,]+ + ) + )? + ( + \s*\bCOMMENT\b\s+( + \'(\\\'|[^\']|,)+\' + | + \"(\\\"|[^\"]|,)+\" + | + [^,]+ + ) + )? + ) + """ + + _COLUMN_CONSTRAINT = re.sub(r"(^\s+|\n)", r"", _COLUMN_CONSTRAINT_BASE, flags=re.MULTILINE) + + _CREATE_TABLE_STATEMENT = Suppress(_CREATE) + Optional(_TEMP)("temp") + Suppress(_TABLE) + Optional(Suppress(CaselessKeyword("IF NOT EXISTS"))) \ + Optional(_SUPPRESS_QUOTE) + Optional(Word(alphanums+"_")("schema") + Optional(_SUPPRESS_QUOTE) + _DOT + Optional(_SUPPRESS_QUOTE)) + Word(alphanums+"_<>")("table") + Optional(_SUPPRESS_QUOTE) \ + _LPAR \ @@ -531,7 +568,7 @@ class DdlParse(DdlParseBase): + Optional(_LPAR + Regex(r"[\d\*]+\s*,*\s*\d*") + Optional(Suppress(_CHAR_SEMANTICS | _BYTE_SEMANTICS)) + _RPAR) )("type") + Optional(Word(r"\[\]"))("array_brackets") - + Optional(Regex(r"(?!--)(\b(COMMENT|DEFAULT)\b\s+[^,]+|([A-Za-z0-9_\.'\": -]|[^\x01-\x7E])*)", re.IGNORECASE))("constraint") + + Optional(Regex(_COLUMN_CONSTRAINT, re.IGNORECASE))("constraint") )("column") | _COMMENT diff --git a/setup.py b/setup.py index 438beab..bf722c3 100644 --- a/setup.py +++ b/setup.py @@ -60,10 +60,10 @@ def _test_requirements(): 'License :: OSI Approved :: BSD License', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Topic :: Database', 'Topic :: Software Development :: Libraries :: Python Modules', ], diff --git a/test/test_ddlparse.py b/test/test_ddlparse.py index b8d32de..9ab1843 100644 --- a/test/test_ddlparse.py +++ b/test/test_ddlparse.py @@ -326,7 +326,8 @@ Col_03 integer DEFAULT 0, Col_04 numeric(10) DEFAULT 0, Col_05 numeric(20,3) DEFAULT 0.0, - Col_06 character varying[] DEFAULT '{}'::character varying[] + Col_06 varchar(100) DEFAULT '!\"#$%&\\\'()*+,-./:;<=>?@[\\]^_`{|}~', + Col_07 character varying[] DEFAULT '{}'::character varying[] ); """, "database" : None, @@ -337,7 +338,8 @@ {"name" : "Col_03", "type" : "INTEGER", "length" : None, "scale" : None, "array_dimensional" : 0, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : None}, {"name" : "Col_04", "type" : "NUMERIC", "length" : 10, "scale" : None, "array_dimensional" : 0, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : None}, {"name" : "Col_05", "type" : "NUMERIC", "length" : 20, "scale" : 3, "array_dimensional" : 0, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : None}, - {"name" : "Col_06", "type" : "CHARACTER VARYING", "length" : None, "scale" : None, "array_dimensional" : 1, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : None}, + {"name" : "Col_06", "type" : "VARCHAR", "length" : 100, "scale" : None, "array_dimensional" : 0, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : None}, + {"name" : "Col_07", "type" : "CHARACTER VARYING", "length" : None, "scale" : None, "array_dimensional" : 1, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : None}, ], "bq_field" : [ '{"name": "Col_01", "type": "STRING", "mode": "NULLABLE"}', @@ -345,7 +347,8 @@ '{"name": "Col_03", "type": "INTEGER", "mode": "NULLABLE"}', '{"name": "Col_04", "type": "INTEGER", "mode": "NULLABLE"}', '{"name": "Col_05", "type": "NUMERIC", "mode": "NULLABLE"}', - '{"name": "Col_06", "type": "STRING", "mode": "REPEATED"}', + '{"name": "Col_06", "type": "STRING", "mode": "NULLABLE"}', + '{"name": "Col_07", "type": "STRING", "mode": "REPEATED"}', ], "bq_standard_data_type" : [ "STRING", @@ -354,6 +357,7 @@ "INT64", "NUMERIC", "STRING", + "STRING", ], }, @@ -588,7 +592,8 @@ Col_03 integer[] COMMENT 'in "Quote"', -- one dimensional array Col_04 integer[][] COMMENT "in 'Quote'", -- two dimensional array Col_05 integer[][][] COMMENT 'コメント is full-width(Japanese) character', -- multiple dimensional array - Col_06 character varying[] -- COMMENT 'Comment out' -- character varying array + Col_06 character NOT NULL DEFAULT 'a\\'bc' COMMENT 'Comma, strings, ,', -- Comma, strings, , + Col_07 character varying[] -- COMMENT 'Comment out' -- character varying array ); """, "database" : None, @@ -599,7 +604,8 @@ {"name" : "Col_03", "type" : "INTEGER", "length" : None, "scale" : None, "array_dimensional" : 1, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : "in \"Quote\""}, {"name" : "Col_04", "type" : "INTEGER", "length" : None, "scale" : None, "array_dimensional" : 2, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : "in 'Quote'"}, {"name" : "Col_05", "type" : "INTEGER", "length" : None, "scale" : None, "array_dimensional" : 3, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : "コメント is full-width(Japanese) character"}, - {"name" : "Col_06", "type" : "CHARACTER VARYING", "length" : None, "scale" : None, "array_dimensional" : 1, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : None}, + {"name" : "Col_06", "type" : "CHARACTER", "length" : None, "scale" : None, "array_dimensional" : 0, "not_null" : True, "pk" : False, "unique" : False, "constraint" : "NOT NULL", "description" : "Comma, strings, ,"}, + {"name" : "Col_07", "type" : "CHARACTER VARYING", "length" : None, "scale" : None, "array_dimensional" : 1, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : None}, ], "bq_field" : [ '{"name": "Col_01", "type": "STRING", "mode": "REQUIRED", "description": "Single Quote"}', @@ -607,7 +613,8 @@ '{"name": "Col_03", "type": "INTEGER", "mode": "REPEATED", "description": "in \\"Quote\\""}', '{"name": "Col_04", "type": "RECORD", "mode": "REPEATED", "description": "in \'Quote\'", "fields": [{"name": "dimension_1", "type": "INTEGER", "mode": "REPEATED"}]}', '{"name": "Col_05", "type": "RECORD", "mode": "REPEATED", "description": "コメント is full-width(Japanese) character", "fields": [{"name": "dimension_1", "type": "RECORD", "mode": "REPEATED", "fields": [{"name": "dimension_2", "type": "INTEGER", "mode": "REPEATED"}]}]}', - '{"name": "Col_06", "type": "STRING", "mode": "REPEATED"}', + '{"name": "Col_06", "type": "STRING", "mode": "REQUIRED", "description": "Comma, strings, ,"}', + '{"name": "Col_07", "type": "STRING", "mode": "REPEATED"}', ], "bq_standard_data_type" : [ "STRING", @@ -616,6 +623,7 @@ "INT64", "INT64", "STRING", + "STRING", ], }, } @@ -784,7 +792,8 @@ Col_03 integer[] COMMENT 'in "Quote"', -- one dimensional array Col_04 integer[][] COMMENT "in 'Quote'", -- two dimensional array Col_05 integer[][][] COMMENT 'コメント is full-width(Japanese) character', -- multiple dimensional array - Col_06 character varying[] -- COMMENT 'Comment out' -- character varying array + Col_06 character COMMENT 'Comma, strings, ,', -- Comma, strings, , + Col_07 character varying[] -- COMMENT 'Comment out' -- character varying array ); """, "bq_ddl": { @@ -798,7 +807,8 @@ Col_03 ARRAY OPTIONS (description = "in \\"Quote\\""), Col_04 ARRAY>> OPTIONS (description = "in 'Quote'"), Col_05 ARRAY>>>> OPTIONS (description = "コメント is full-width(Japanese) character"), - Col_06 ARRAY + Col_06 STRING OPTIONS (description = "Comma, strings, ,"), + Col_07 ARRAY )""", DdlParse.NAME_CASE.lower: """\ @@ -810,7 +820,8 @@ col_03 ARRAY OPTIONS (description = "in \\"Quote\\""), col_04 ARRAY>> OPTIONS (description = "in 'Quote'"), col_05 ARRAY>>>> OPTIONS (description = "コメント is full-width(Japanese) character"), - col_06 ARRAY + col_06 STRING OPTIONS (description = "Comma, strings, ,"), + col_07 ARRAY )""", DdlParse.NAME_CASE.upper: """\ @@ -822,7 +833,8 @@ COL_03 ARRAY OPTIONS (description = "in \\"Quote\\""), COL_04 ARRAY>> OPTIONS (description = "in 'Quote'"), COL_05 ARRAY>>>> OPTIONS (description = "コメント is full-width(Japanese) character"), - COL_06 ARRAY + COL_06 STRING OPTIONS (description = "Comma, strings, ,"), + COL_07 ARRAY )""", }, }, diff --git a/tox.ini b/tox.ini index 2d5ff26..0241569 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py34,py35,py36,py37 +envlist = py35,py36,py37,py38 [testenv] deps= From a14c5cfdb4f81478d74fb6caf0f4ddc52cea5e12 Mon Sep 17 00:00:00 2001 From: shinichi-takii Date: Mon, 6 Jul 2020 13:18:08 +0900 Subject: [PATCH 2/3] Fix version --- test-requirements.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index bae867f..b02141f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,6 @@ -pytest>=3.6 -pytest-cov -tox -coveralls -codecov -codeclimate-test-reporter +pytest>=5.4.3 +pytest-cov>=2.10.0 +tox>=3.16.1 +coveralls>=2.0.0 +codecov>=2.1.7 +codeclimate-test-reporter>=0.2.3 From 10c5a02d46f4c71acbd1fe7af7082d5ad728ed20 Mon Sep 17 00:00:00 2001 From: shinichi-takii Date: Mon, 6 Jul 2020 13:26:00 +0900 Subject: [PATCH 3/3] Fix parse for column-constraint --- ddlparse/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ddlparse/__init__.py b/ddlparse/__init__.py index 89272aa..1f81563 100644 --- a/ddlparse/__init__.py +++ b/ddlparse/__init__.py @@ -7,8 +7,8 @@ from .ddlparse import * -__copyright__ = 'Copyright (C) 2018-2019 Shinichi Takii' -__version__ = '1.4.0' +__copyright__ = 'Copyright (C) 2018-2020 Shinichi Takii' +__version__ = '1.5.0' __license__ = 'BSD-3-Clause' __author__ = 'Shinichi Takii' __author_email__ = 'shinichi.takii@gmail.com'