Merge pull request #49 from shinichi-takii/feature/fix-parse-column-c…

…onstraint Fix parse for column-constraint
shinichi-takii · Jul 6, 2020 · a581a27 · a581a27
2 parents f8f7c0d + 10c5a02
commit a581a27
Show file tree

Hide file tree

Showing 11 changed files with 110 additions and 24 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,2 +1,23 @@
-# Changes
+## Summary
+<!-- Changelog format : https://keepachangelog.com/ -->
+
+### Added
+- {{ added-summary }}
+
+### Changed
+- {{ changed-summary }}
+
+### Removed
+- {{ removed-summary }}
+
+### Fixed
+- {{ fixed-summary }}
+
+
+## File Details
+### {{ file-name }}
+- {{ summary }}
+
+
+## Applicable Issues
 - fix # : summary...
diff --git a/.gitignore b/.gitignore
@@ -2,6 +2,9 @@
 .DS_Store
 *~
 
+# IDE
+.vscode
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

diff --git a/.travis.yml b/.travis.yml
@@ -3,10 +3,10 @@ dist: xenial
 sudo: true
 
 python:
-  - "3.4"
   - "3.5"
   - "3.6"
   - "3.7"
+  - "3.8"
 
 # command to install dependencies
 install:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+## [1.5.0] - 2020-07-06
+### Added
+- Add supports for Python 3.8
+
+### Removed
+- End of support for Python 3.4
+
+### Fixed
+- Fix parse for column-constraint.
+- Miner fix.
+
+
 ## [1.4.0] - 2019-12-08
 ### Added
 - Add supports to BigQuery `NUMERIC` data type.
@@ -130,6 +142,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Initial released.
 
 
+[1.5.0]: https://github.com/shinichi-takii/ddlparse/compare/v1.4.0...v1.5.0
 [1.4.0]: https://github.com/shinichi-takii/ddlparse/compare/v1.3.1...v1.4.0
 [1.3.1]: https://github.com/shinichi-takii/ddlparse/compare/v1.3.0...v1.3.1
 [1.3.0]: https://github.com/shinichi-takii/ddlparse/compare/v1.2.3...v1.3.0

diff --git a/README.md b/README.md
@@ -21,7 +21,7 @@
 
 ## Requirement
 
-1. Python >= 3.4
+1. Python >= 3.5
 1. [pyparsing](https://github.com/pyparsing/pyparsing)
 
 ## Installation

diff --git a/ddlparse/__init__.py b/ddlparse/__init__.py
@@ -7,8 +7,8 @@
 
 from .ddlparse import *
 
-__copyright__    = 'Copyright (C) 2018-2019 Shinichi Takii'
-__version__      = '1.4.0'
+__copyright__    = 'Copyright (C) 2018-2020 Shinichi Takii'
+__version__      = '1.5.0'
 __license__      = 'BSD-3-Clause'
 __author__       = 'Shinichi Takii'
 __author_email__ = '[email protected]'

diff --git a/ddlparse/ddlparse.py b/ddlparse/ddlparse.py
@@ -493,6 +493,43 @@ class DdlParse(DdlParseBase):
 
     _COMMENT = Suppress("--" + Regex(r".+"))
 
+    _COLUMN_CONSTRAINT_BASE = r"""
+    (?!--)
+    (
+        (
+            \s*\b(?:NOT\s+)NULL?\b
+        )?
+        (
+            \s*\bAUTO_INCREMENT\b
+        )?(
+            \s*\b(UNIQUE|PRIMARY)(?:\s+KEY)?\b
+        )?
+        (
+            \s*\bDEFAULT\b\s+(
+                ([A-Za-z0-9_\.\'\" -]|[^\x01-\x7E])*\:\:[A-Za-z0-9\[\]]+
+                |
+                \'(\\\'|[^\']|,)+\'
+                |
+                \"(\\\"|[^\"]|,)+\"
+                |
+                [^,]+
+            )
+        )?
+        (
+            \s*\bCOMMENT\b\s+(
+                \'(\\\'|[^\']|,)+\'
+                |
+                \"(\\\"|[^\"]|,)+\"
+                |
+                [^,]+
+            )
+        )?
+    )
+    """
+
+    _COLUMN_CONSTRAINT = re.sub(r"(^\s+|\n)", r"", _COLUMN_CONSTRAINT_BASE, flags=re.MULTILINE)
+
+
     _CREATE_TABLE_STATEMENT = Suppress(_CREATE) + Optional(_TEMP)("temp") + Suppress(_TABLE) + Optional(Suppress(CaselessKeyword("IF NOT EXISTS"))) \
         + Optional(_SUPPRESS_QUOTE) + Optional(Word(alphanums+"_")("schema") + Optional(_SUPPRESS_QUOTE) + _DOT + Optional(_SUPPRESS_QUOTE)) + Word(alphanums+"_<>")("table") + Optional(_SUPPRESS_QUOTE) \
         + _LPAR \
@@ -531,7 +568,7 @@ class DdlParse(DdlParseBase):
                         + Optional(_LPAR + Regex(r"[\d\*]+\s*,*\s*\d*") + Optional(Suppress(_CHAR_SEMANTICS | _BYTE_SEMANTICS)) + _RPAR)
                         )("type")
                     + Optional(Word(r"\[\]"))("array_brackets")
-                    + Optional(Regex(r"(?!--)(\b(COMMENT|DEFAULT)\b\s+[^,]+|([A-Za-z0-9_\.'\": -]|[^\x01-\x7E])*)", re.IGNORECASE))("constraint")
+                    + Optional(Regex(_COLUMN_CONSTRAINT, re.IGNORECASE))("constraint")
                 )("column")
                 |
                 _COMMENT

diff --git a/setup.py b/setup.py
@@ -60,10 +60,10 @@ def _test_requirements():
         'License :: OSI Approved :: BSD License',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Topic :: Database',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],

diff --git a/test-requirements.txt b/test-requirements.txt
@@ -1,6 +1,6 @@
-pytest>=3.6
-pytest-cov
-tox
-coveralls
-codecov
-codeclimate-test-reporter
+pytest>=5.4.3
+pytest-cov>=2.10.0
+tox>=3.16.1
+coveralls>=2.0.0
+codecov>=2.1.7
+codeclimate-test-reporter>=0.2.3
diff --git a/test/test_ddlparse.py b/test/test_ddlparse.py
@@ -326,7 +326,8 @@
               Col_03 integer DEFAULT 0,
               Col_04 numeric(10) DEFAULT 0,
               Col_05 numeric(20,3) DEFAULT 0.0,
-              Col_06 character varying[] DEFAULT '{}'::character varying[]
+              Col_06 varchar(100) DEFAULT '!\"#$%&\\\'()*+,-./:;<=>?@[\\]^_`{|}~',
+              Col_07 character varying[] DEFAULT '{}'::character varying[]
             );
             """,
         "database" : None,
@@ -337,15 +338,17 @@
             {"name" : "Col_03", "type" : "INTEGER", "length" : None, "scale" : None, "array_dimensional" : 0, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : None},
             {"name" : "Col_04", "type" : "NUMERIC", "length" : 10, "scale" : None, "array_dimensional" : 0, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : None},
             {"name" : "Col_05", "type" : "NUMERIC", "length" : 20, "scale" : 3, "array_dimensional" : 0, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : None},
-            {"name" : "Col_06", "type" : "CHARACTER VARYING", "length" : None, "scale" : None, "array_dimensional" : 1, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : None},
+            {"name" : "Col_06", "type" : "VARCHAR", "length" : 100, "scale" : None, "array_dimensional" : 0, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : None},
+            {"name" : "Col_07", "type" : "CHARACTER VARYING", "length" : None, "scale" : None, "array_dimensional" : 1, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : None},
         ],
         "bq_field" : [
             '{"name": "Col_01", "type": "STRING", "mode": "NULLABLE"}',
             '{"name": "Col_02", "type": "STRING", "mode": "NULLABLE"}',
             '{"name": "Col_03", "type": "INTEGER", "mode": "NULLABLE"}',
             '{"name": "Col_04", "type": "INTEGER", "mode": "NULLABLE"}',
             '{"name": "Col_05", "type": "NUMERIC", "mode": "NULLABLE"}',
-            '{"name": "Col_06", "type": "STRING", "mode": "REPEATED"}',
+            '{"name": "Col_06", "type": "STRING", "mode": "NULLABLE"}',
+            '{"name": "Col_07", "type": "STRING", "mode": "REPEATED"}',
         ],
         "bq_standard_data_type" : [
             "STRING",
@@ -354,6 +357,7 @@
             "INT64",
             "NUMERIC",
             "STRING",
+            "STRING",
         ],
     },
 
@@ -588,7 +592,8 @@
               Col_03 integer[] COMMENT 'in "Quote"', -- one dimensional array
               Col_04 integer[][] COMMENT "in 'Quote'", -- two dimensional array
               Col_05 integer[][][] COMMENT 'コメント is full-width(Japanese) character', -- multiple dimensional array
-              Col_06 character varying[] -- COMMENT 'Comment out' -- character varying array
+              Col_06 character NOT NULL DEFAULT 'a\\'bc' COMMENT 'Comma, strings, ,', -- Comma, strings, ,
+              Col_07 character varying[] -- COMMENT 'Comment out' -- character varying array
             );
             """,
         "database" : None,
@@ -599,15 +604,17 @@
             {"name" : "Col_03", "type" : "INTEGER", "length" : None, "scale" : None, "array_dimensional" : 1, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : "in \"Quote\""},
             {"name" : "Col_04", "type" : "INTEGER", "length" : None, "scale" : None, "array_dimensional" : 2, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : "in 'Quote'"},
             {"name" : "Col_05", "type" : "INTEGER", "length" : None, "scale" : None, "array_dimensional" : 3, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : "コメント is full-width(Japanese) character"},
-            {"name" : "Col_06", "type" : "CHARACTER VARYING", "length" : None, "scale" : None, "array_dimensional" : 1, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : None},
+            {"name" : "Col_06", "type" : "CHARACTER", "length" : None, "scale" : None, "array_dimensional" : 0, "not_null" : True, "pk" : False, "unique" : False, "constraint" : "NOT NULL", "description" : "Comma, strings, ,"},
+            {"name" : "Col_07", "type" : "CHARACTER VARYING", "length" : None, "scale" : None, "array_dimensional" : 1, "not_null" : False, "pk" : False, "unique" : False, "constraint" : "", "description" : None},
         ],
         "bq_field" : [
             '{"name": "Col_01", "type": "STRING", "mode": "REQUIRED", "description": "Single Quote"}',
             '{"name": "Col_02", "type": "STRING", "mode": "REQUIRED", "description": "Double Quote"}',
             '{"name": "Col_03", "type": "INTEGER", "mode": "REPEATED", "description": "in \\"Quote\\""}',
             '{"name": "Col_04", "type": "RECORD", "mode": "REPEATED", "description": "in \'Quote\'", "fields": [{"name": "dimension_1", "type": "INTEGER", "mode": "REPEATED"}]}',
             '{"name": "Col_05", "type": "RECORD", "mode": "REPEATED", "description": "コメント is full-width(Japanese) character", "fields": [{"name": "dimension_1", "type": "RECORD", "mode": "REPEATED", "fields": [{"name": "dimension_2", "type": "INTEGER", "mode": "REPEATED"}]}]}',
-            '{"name": "Col_06", "type": "STRING", "mode": "REPEATED"}',
+            '{"name": "Col_06", "type": "STRING", "mode": "REQUIRED", "description": "Comma, strings, ,"}',
+            '{"name": "Col_07", "type": "STRING", "mode": "REPEATED"}',
         ],
         "bq_standard_data_type" : [
             "STRING",
@@ -616,6 +623,7 @@
             "INT64",
             "INT64",
             "STRING",
+            "STRING",
         ],
     },
 }
@@ -784,7 +792,8 @@
               Col_03 integer[] COMMENT 'in "Quote"', -- one dimensional array
               Col_04 integer[][] COMMENT "in 'Quote'", -- two dimensional array
               Col_05 integer[][][] COMMENT 'コメント is full-width(Japanese) character', -- multiple dimensional array
-              Col_06 character varying[] -- COMMENT 'Comment out' -- character varying array
+              Col_06 character COMMENT 'Comma, strings, ,', -- Comma, strings, ,
+              Col_07 character varying[] -- COMMENT 'Comment out' -- character varying array
             );
             """,
         "bq_ddl": {
@@ -798,7 +807,8 @@
                   Col_03 ARRAY<INT64> OPTIONS (description = "in \\"Quote\\""),
                   Col_04 ARRAY<STRUCT<dimension_1 ARRAY<INT64>>> OPTIONS (description = "in 'Quote'"),
                   Col_05 ARRAY<STRUCT<dimension_1 ARRAY<STRUCT<dimension_2 ARRAY<INT64>>>>> OPTIONS (description = "コメント is full-width(Japanese) character"),
-                  Col_06 ARRAY<STRING>
+                  Col_06 STRING OPTIONS (description = "Comma, strings, ,"),
+                  Col_07 ARRAY<STRING>
                 )""",
             DdlParse.NAME_CASE.lower:
                 """\
@@ -810,7 +820,8 @@
                   col_03 ARRAY<INT64> OPTIONS (description = "in \\"Quote\\""),
                   col_04 ARRAY<STRUCT<dimension_1 ARRAY<INT64>>> OPTIONS (description = "in 'Quote'"),
                   col_05 ARRAY<STRUCT<dimension_1 ARRAY<STRUCT<dimension_2 ARRAY<INT64>>>>> OPTIONS (description = "コメント is full-width(Japanese) character"),
-                  col_06 ARRAY<STRING>
+                  col_06 STRING OPTIONS (description = "Comma, strings, ,"),
+                  col_07 ARRAY<STRING>
                 )""",
             DdlParse.NAME_CASE.upper:
                 """\
@@ -822,7 +833,8 @@
                   COL_03 ARRAY<INT64> OPTIONS (description = "in \\"Quote\\""),
                   COL_04 ARRAY<STRUCT<dimension_1 ARRAY<INT64>>> OPTIONS (description = "in 'Quote'"),
                   COL_05 ARRAY<STRUCT<dimension_1 ARRAY<STRUCT<dimension_2 ARRAY<INT64>>>>> OPTIONS (description = "コメント is full-width(Japanese) character"),
-                  COL_06 ARRAY<STRING>
+                  COL_06 STRING OPTIONS (description = "Comma, strings, ,"),
+                  COL_07 ARRAY<STRING>
                 )""",
         },
     },

diff --git a/tox.ini b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = py34,py35,py36,py37
+envlist = py35,py36,py37,py38
 
 [testenv]
 deps=
-Original file line number
+Diff line change
@@ Expand Up / @@ -3,10 +3,10 @@ dist: xenial @@
     sudo: true
     python:
-      - "3.4"
       - "3.5"
       - "3.6"
       - "3.7"
+      - "3.8"
     # command to install dependencies
     install:
@@ Expand Down @@