diff --git a/poetry.lock b/poetry.lock index 1f4f8e38..cfb574bd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -620,20 +620,20 @@ files = [ [[package]] name = "sqlglot" -version = "30.21.0" +version = "30.22.0" description = "An easily customizable SQL parser and transpiler" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "sqlglot-30.21.0-py3-none-any.whl", hash = "sha256:816d1a4815b7562b3976efcc0b561c928a743f976efd0fb8ad6db7a5b96c069e"}, - {file = "sqlglot-30.21.0.tar.gz", hash = "sha256:6787ac202a3adc0e25434f1512370a655e51cb3de8c792544de5f6ff9c618d25"}, + {file = "sqlglot-30.22.0-py3-none-any.whl", hash = "sha256:90aa461490fcd95d14ec3842a97506ae20f6d3e9313307ad31be793d479cca65"}, + {file = "sqlglot-30.22.0.tar.gz", hash = "sha256:ec4b83ca8236ea8867f574a382dc15ce35b071c977fecfcc66482d9a3f500661"}, ] [package.extras] -c = ["sqlglotc (==30.21.0) ; python_version >= \"3.10\""] -dev = ["duckdb (>=0.6)", "mypy ; python_version < \"3.10\"", "pandas", "pandas-stubs", "pdoc", "pre-commit", "pyperf", "python-dateutil", "pytz", "ruff (==0.15.6)", "setuptools_scm", "sqlglot-mypy (>=2.3.0.post2) ; python_version >= \"3.10\"", "types-python-dateutil", "types-pytz", "typing_extensions"] -rs = ["sqlglotc (==30.21.0) ; python_version >= \"3.10\"", "sqlglotrs (==0.13.0)"] +c = ["sqlglotc (==30.22.0) ; python_version >= \"3.10\""] +dev = ["duckdb (>=0.6)", "mypy (>=2.4.0) ; python_version >= \"3.10\"", "mypy ; python_version < \"3.10\"", "pandas", "pandas-stubs", "pdoc", "pre-commit", "pyperf", "python-dateutil", "pytz", "ruff (==0.15.6)", "setuptools_scm", "types-python-dateutil", "types-pytz", "typing_extensions"] +rs = ["sqlglotc (==30.22.0) ; python_version >= \"3.10\"", "sqlglotrs (==0.13.0)"] [[package]] name = "tomli" diff --git a/sql_metadata/comments.py b/sql_metadata/comments.py index a5fc915c..57c2b1cf 100644 --- a/sql_metadata/comments.py +++ b/sql_metadata/comments.py @@ -21,10 +21,23 @@ import re from typing import Any +from sqlglot.dialects.mysql import MySQL from sqlglot.errors import TokenError from sqlglot.tokens import Tokenizer +class _MySQLCommentTokenizer(MySQL.Tokenizer): + """MySQL tokenizer that treats any ``--`` as a line comment. + + sqlglot >=30.22 follows MySQL semantics where ``--`` only starts a + comment when followed by whitespace, so ``--foo`` would be tokenized + as two dashes and an identifier. We use this tokenizer purely to + strip comments, and ``--foo`` is meant as a comment in practice. + """ + + DASH_COMMENT_REQUIRES_BOUNDARY = False + + def _choose_tokenizer(sql: str) -> Tokenizer: """Select the appropriate sqlglot tokenizer for *sql*. @@ -39,13 +52,11 @@ def _choose_tokenizer(sql: str) -> Tokenizer: :rtype: sqlglot.tokens.Tokenizer """ if "#" in sql and not _has_hash_variables(sql): - from sqlglot.dialects.mysql import MySQL - # Pass dialect="mysql" so the cached TokenizerCore (sqlglot >=30.7.0 # caches per-class) is built with MySQL identifier semantics — without # it the cache is primed from the default dialect and later mysql # parses misclassify e.g. ``0020_big_table`` as NUMBER + VAR. - return MySQL.Tokenizer(dialect="mysql") + return _MySQLCommentTokenizer(dialect="mysql") return Tokenizer() @@ -180,12 +191,10 @@ def strip_comments_for_parsing(sql: str) -> str: if upper.startswith("CREATE FUNCTION") or _has_hash_variables(sql): tokenizer = Tokenizer() else: - from sqlglot.dialects.mysql import MySQL - # See _choose_tokenizer — the explicit dialect prevents the # sqlglot >=30.7.0 TokenizerCore cache from being primed with # default-dialect semantics. - tokenizer = MySQL.Tokenizer(dialect="mysql") + tokenizer = _MySQLCommentTokenizer(dialect="mysql") try: tokens = list(tokenizer.tokenize(sql)) except TokenError: