pyproject.toml

[project]
name = "databricks-labs-remorph"
description = 'SQL code converter and data reconcilation tool for accelerating data onboarding to Databricks from EDW, CDW and other ETL sources.'
license-files = { paths = ["LICENSE", "NOTICE"] }
keywords = ["Databricks"]
readme = "README.md"
requires-python = ">=3.10"
dynamic = ["version"]
classifiers = [
  "Development Status :: 3 - Alpha",
  "Programming Language :: Python :: 3.10",
  "Programming Language :: Python :: 3.11",
  "Programming Language :: Python :: Implementation :: CPython",
]
dependencies = [
  "databricks-sdk~=0.29.0",
  "sqlglot==25.8.1",
  "databricks-labs-blueprint[yaml]>=0.2.3",
  "databricks-labs-lsql>=0.4.3",
]

[project.urls]
Documentation = "https://github.com/databrickslabs/remorph"
Issues = "https://github.com/databrickslabs/remorph/issues"
Source = "https://github.com/databrickslabs/remorph"

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[tool.hatch.build]
sources = ["src"]
include = ["src"]

[tool.hatch.version]
path = "src/databricks/labs/remorph/__about__.py"

[tool.hatch.envs.default]
python="3.10"

# store virtual env as the child of this folder. Helps VSCode to run better
path = ".venv"

dependencies = [
  "pylint",
  "pylint-pytest==2.0.0a0",
  "coverage[toml]>=6.5",
  "pytest",
  "pytest-cov>=4.0.0,<5.0.0",
  "black>=23.1.0",
  "ruff>=0.0.243",
  "databricks-connect==15.1",
  "types-pyYAML",
  "types-pytz",
  "databricks-labs-pylint~=0.4.0",
  "mypy~=1.10.0",
  "numpy==1.26.4",
]

[project.entry-points.databricks]
reconcile = "databricks.labs.remorph.reconcile.execute:main"

[tool.hatch.envs.default.scripts]
test        = "pytest --cov src --cov-report=xml tests/unit"
coverage    = "pytest --cov src tests/unit --cov-report=html"
integration = "pytest --cov src tests/integration --durations 20"
fmt         = ["black .",
               "ruff  check . --fix",
               "mypy --disable-error-code 'annotation-unchecked' .",
               "pylint --output-format=colorized -j 0 src tests"]
verify      = ["black --check .",
               "ruff check .",
               "mypy --disable-error-code 'annotation-unchecked' .",
               "pylint --output-format=colorized -j 0 src tests"]

[tool.hatch.envs.sqlglot-latest]
python="3.10"
dependencies = [
  "databricks-labs-remorph",
  "sqlglot",
]


[tool.pytest.ini_options]
addopts = "-s -p no:warnings -vv --cache-clear"
cache_dir = ".venv/pytest-cache"

[tool.black]
target-version = ["py310"]
line-length = 120
skip-string-normalization = true

[tool.ruff]
cache-dir = ".venv/ruff-cache"
target-version = "py310"
line-length = 120

lint.ignore = [
  # Allow non-abstract empty methods in abstract base classes
  "B027",
  # Allow boolean positional values in function calls, like `dict.get(... True)`
  "FBT003",
  # Ignore checks for possible passwords and SQL statement construction
  "S105", "S106", "S107", "S603", "S608",
  # Allow print statements
  "T201",
  # Allow asserts
  "S101",
  # Allow standard random generators
  "S311",
  # Ignore complexity
  "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915",
  # Ignore Exception must not use a string literal, assign to variable first
  "EM101",
  "PLR2004",
  "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)`
]
extend-exclude = [
  "notebooks/*.py"
]


[tool.ruff.lint.flake8-tidy-imports]
ban-relative-imports = "all"

[tool.ruff.lint.per-file-ignores]

"tests/**/*" = [
  "PLR2004", "S101", "TID252", # tests can use magic values, assertions, and relative imports
  "ARG001" # tests may not use the provided fixtures
]

[tool.coverage.run]
branch = true
parallel = true

[tool.coverage.report]
omit = ["src/databricks/labs/remorph/coverage/*",
  "src/databricks/labs/remorph/helpers/execution_time.py",
  "__about__.py"]
exclude_lines = [
  "no cov",
  "if __name__ == .__main__.:",
  "if TYPE_CHECKING:",
  "def main()",
]

[tool.pylint.main]
# PyLint configuration is adapted from Google Python Style Guide with modifications.
# Sources https://google.github.io/styleguide/pylintrc
# License: https://github.com/google/styleguide/blob/gh-pages/LICENSE

# Clear in-memory caches upon conclusion of linting. Useful if running pylint in
# a server-like mode.
# clear-cache-post-run =

# Always return a 0 (non-error) status code, even if lint errors are found. This
# is primarily useful in continuous integration scripts.
# exit-zero =

# A comma-separated list of package or module names from where C extensions may
# be loaded. Extensions are loading into the active Python interpreter and may
# run arbitrary code.
# extension-pkg-allow-list =

# A comma-separated list of package or module names from where C extensions may
# be loaded. Extensions are loading into the active Python interpreter and may
# run arbitrary code. (This is an alternative name to extension-pkg-allow-list
# for backward compatibility.)
# extension-pkg-whitelist =

# Specify a score threshold under which the program will exit with error.
fail-under = 10.0

# Interpret the stdin as a python script, whose filename needs to be passed as
# the module_or_package argument.
# from-stdin =

# Add files or directories matching the regular expressions patterns to the
# ignore-list. The regex matches against paths and can be in Posix or Windows
# format. Because '\\' represents the directory delimiter on Windows systems, it
# can't be used as an escape character.
# ignore-paths =

# Files or directories matching the regular expression patterns are skipped. The
# regex matches against base names, not paths. The default value ignores Emacs
# file locks
ignore-patterns = ["^\\.#"]

# List of module names for which member attributes should not be checked (useful
# for modules/projects where namespaces are manipulated during runtime and thus
# existing member attributes cannot be deduced by static analysis). It supports
# qualified module names, as well as Unix pattern matching.
# ignored-modules =

# Python code to execute, usually for sys.path manipulation such as
# pygtk.require().
# init-hook =

# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
# number of processors available to use, and will cap the count on Windows to
# avoid hangs.
jobs = 0

# Control the amount of potential inferred values when inferring a single object.
# This can help the performance when dealing with large functions or complex,
# nested conditions.
limit-inference-results = 100

# List of plugins (as comma separated values of python module names) to load,
# usually to register additional checkers.
load-plugins = [
  "pylint_pytest",
  "pylint.extensions.bad_builtin",
  "pylint.extensions.broad_try_clause",
  "pylint.extensions.check_elif",
  "pylint.extensions.code_style",
  "pylint.extensions.confusing_elif",
  "pylint.extensions.comparison_placement",
  "pylint.extensions.consider_refactoring_into_while_condition",
  "pylint.extensions.dict_init_mutate",
  "pylint.extensions.docparams",
  "pylint.extensions.dunder",
  "pylint.extensions.for_any_all",
  "pylint.extensions.mccabe",
  "pylint.extensions.overlapping_exceptions",
  "pylint.extensions.private_import",
  "pylint.extensions.redefined_variable_type",
  "pylint.extensions.set_membership",
  "pylint.extensions.typing",
]

# Pickle collected data for later comparisons.
persistent = true

# Minimum Python version to use for version dependent checks. Will default to the
# version used to run pylint.
py-version = "3.10"

# Discover python modules and packages in the file system subtree.
# recursive =

# Add paths to the list of the source roots. Supports globbing patterns. The
# source root is an absolute path or a path relative to the current working
# directory used to determine a package namespace for modules located under the
# source root.
# source-roots =

# When enabled, pylint would attempt to guess common misconfiguration and emit
# user-friendly hints instead of false-positive error messages.
suggestion-mode = true

# Allow loading of arbitrary C extensions. Extensions are imported into the
# active Python interpreter and may run arbitrary code.
# unsafe-load-any-extension =

[tool.pylint.basic]
# Naming style matching correct argument names.
argument-naming-style = "snake_case"

# Regular expression matching correct argument names. Overrides argument-naming-
# style. If left empty, argument names will be checked with the set naming style.
argument-rgx = "[a-z_][a-z0-9_]{2,30}$"

# Naming style matching correct attribute names.
attr-naming-style = "snake_case"

# Regular expression matching correct attribute names. Overrides attr-naming-
# style. If left empty, attribute names will be checked with the set naming
# style.
attr-rgx = "[a-z_][a-z0-9_]{2,}$"

# Bad variable names which should always be refused, separated by a comma.
bad-names = ["foo", "bar", "baz", "toto", "tutu", "tata"]

# Bad variable names regexes, separated by a comma. If names match any regex,
# they will always be refused
# bad-names-rgxs =

# Naming style matching correct class attribute names.
class-attribute-naming-style = "any"

# Regular expression matching correct class attribute names. Overrides class-
# attribute-naming-style. If left empty, class attribute names will be checked
# with the set naming style.
class-attribute-rgx = "([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$"

# Naming style matching correct class constant names.
class-const-naming-style = "UPPER_CASE"

# Regular expression matching correct class constant names. Overrides class-
# const-naming-style. If left empty, class constant names will be checked with
# the set naming style.
# class-const-rgx =

# Naming style matching correct class names.
class-naming-style = "PascalCase"

# Regular expression matching correct class names. Overrides class-naming-style.
# If left empty, class names will be checked with the set naming style.
class-rgx = "[A-Z_][a-zA-Z0-9]+$"

# Naming style matching correct constant names.
const-naming-style = "UPPER_CASE"

# Regular expression matching correct constant names. Overrides const-naming-
# style. If left empty, constant names will be checked with the set naming style.
const-rgx = "(([A-Z_][A-Z0-9_]*)|(__.*__))$"

# Minimum line length for functions/classes that require docstrings, shorter ones
# are exempt.
docstring-min-length = -1

# Naming style matching correct function names.
function-naming-style = "snake_case"

# Regular expression matching correct function names. Overrides function-naming-
# style. If left empty, function names will be checked with the set naming style.
function-rgx = "[a-z_][a-z0-9_]{2,}$"

# Good variable names which should always be accepted, separated by a comma.
good-names = [
  "f",            # use for file handles
  "i", "j", "k",  # use for loops
  "df",           # use for pyspark.sql.DataFrame
  "ex", "e",      # use for exceptions
  "fn", "cb",     # use for callbacks
  "_",            # use for ignores
  "a",            # use for databricks.sdk.AccountClient
  "w", "ws"       # use for databricks.sdk.WorkspaceClient
]

# Good variable names regexes, separated by a comma. If names match any regex,
# they will always be accepted
# good-names-rgxs =

# Include a hint for the correct naming format with invalid-name.
# include-naming-hint =

# Naming style matching correct inline iteration names.
inlinevar-naming-style = "any"

# Regular expression matching correct inline iteration names. Overrides
# inlinevar-naming-style. If left empty, inline iteration names will be checked
# with the set naming style.
inlinevar-rgx = "[A-Za-z_][A-Za-z0-9_]*$"

# Naming style matching correct method names.
method-naming-style = "snake_case"

# Regular expression matching correct method names. Overrides method-naming-
# style. If left empty, method names will be checked with the set naming style.
method-rgx = "[a-z_][a-z0-9_]{2,}$"

# Naming style matching correct module names.
module-naming-style = "snake_case"

# Regular expression matching correct module names. Overrides module-naming-
# style. If left empty, module names will be checked with the set naming style.
module-rgx = "(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$"

# Colon-delimited sets of names that determine each other's naming style when the
# name regexes allow several styles.
# name-group =

# Regular expression which should only match function or class names that do not
# require a docstring.
no-docstring-rgx = "__.*__"

# List of decorators that produce properties, such as abc.abstractproperty. Add
# to this list to register other decorators that produce valid properties. These
# decorators are taken in consideration only for invalid-name.
property-classes = ["abc.abstractproperty"]

# Regular expression matching correct type alias names. If left empty, type alias
# names will be checked with the set naming style.
# typealias-rgx =

# Regular expression matching correct type variable names. If left empty, type
# variable names will be checked with the set naming style.
# typevar-rgx =

# Naming style matching correct variable names.
variable-naming-style = "snake_case"

# Regular expression matching correct variable names. Overrides variable-naming-
# style. If left empty, variable names will be checked with the set naming style.
variable-rgx = "[a-z_][a-z0-9_]{2,30}$"

[tool.pylint.broad_try_clause]
# Maximum number of statements allowed in a try clause
max-try-statements = 7

[tool.pylint.classes]
# Warn about protected attribute access inside special methods
# check-protected-access-in-special-methods =

# List of method names used to declare (i.e. assign) instance attributes.
defining-attr-methods = ["__init__", "__new__", "setUp", "__post_init__"]

# List of member names, which should be excluded from the protected access
# warning.
exclude-protected = ["_asdict", "_fields", "_replace", "_source", "_make"]

# List of valid names for the first argument in a class method.
valid-classmethod-first-arg = ["cls"]

# List of valid names for the first argument in a metaclass class method.
valid-metaclass-classmethod-first-arg = ["mcs"]

[tool.pylint.deprecated_builtins]
# List of builtins function names that should not be used, separated by a comma
bad-functions = ["map", "input"]

[tool.pylint.design]
# List of regular expressions of class ancestor names to ignore when counting
# public methods (see R0903)
# exclude-too-few-public-methods =

# List of qualified class names to ignore when counting class parents (see R0901)
# ignored-parents =

# Maximum number of arguments for function / method.
max-args = 9

# Maximum number of attributes for a class (see R0902).
max-attributes = 13

# Maximum number of boolean expressions in an if statement (see R0916).
max-bool-expr = 5

# Maximum number of branch for function / method body.
max-branches = 20

# Maximum number of locals for function / method body.
max-locals = 19

# Maximum number of parents for a class (see R0901).
max-parents = 7

# Maximum number of public methods for a class (see R0904).
max-public-methods = 20

# Maximum number of return / yield for function / method body.
max-returns = 11

# Maximum number of statements in function / method body.
max-statements = 50

# Minimum number of public methods for a class (see R0903).
min-public-methods = 2

[tool.pylint.exceptions]
# Exceptions that will emit a warning when caught.
overgeneral-exceptions = ["builtins.Exception"]

[tool.pylint.format]
# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
# expected-line-ending-format =

# Regexp for a line that is allowed to be longer than the limit.
ignore-long-lines = "^\\s*(# )?<?https?://\\S+>?$"

# Number of spaces of indent required inside a hanging or continued line.
indent-after-paren = 4

# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
# tab).
indent-string = "    "

# Maximum number of characters on a single line.
max-line-length = 100

# Maximum number of lines in a module.
max-module-lines = 2000

# Allow the body of a class to be on the same line as the declaration if body
# contains single statement.
# single-line-class-stmt =

# Allow the body of an if to be on the same line as the test if there is no else.
# single-line-if-stmt =

[tool.pylint.imports]
# List of modules that can be imported at any level, not just the top level one.
# allow-any-import-level =

# Allow explicit reexports by alias from a package __init__.
# allow-reexport-from-package =

# Allow wildcard imports from modules that define __all__.
# allow-wildcard-with-all =

# Deprecated modules which should not be used, separated by a comma.
deprecated-modules = ["regsub", "TERMIOS", "Bastion", "rexec"]

# Output a graph (.gv or any supported image format) of external dependencies to
# the given file (report RP0402 must not be disabled).
# ext-import-graph =

# Output a graph (.gv or any supported image format) of all (i.e. internal and
# external) dependencies to the given file (report RP0402 must not be disabled).
# import-graph =

# Output a graph (.gv or any supported image format) of internal dependencies to
# the given file (report RP0402 must not be disabled).
# int-import-graph =

# Force import order to recognize a module as part of the standard compatibility
# libraries.
# known-standard-library =

# Force import order to recognize a module as part of a third party library.
known-third-party = ["enchant"]

# Couples of modules and preferred modules, separated by a comma.
# preferred-modules =

[tool.pylint.logging]
# The type of string formatting that logging methods do. `old` means using %
# formatting, `new` is for `{}` formatting.
logging-format-style = "new"

# Logging modules to check that the string format arguments are in logging
# function parameter format.
logging-modules = ["logging"]

[tool.pylint."messages control"]
# Only show warnings with the listed confidence levels. Leave empty to show all.
# Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, UNDEFINED.
confidence = ["HIGH", "CONTROL_FLOW", "INFERENCE", "INFERENCE_FAILURE", "UNDEFINED"]

# Disable the message, report, category or checker with the given id(s). You can
# either give multiple identifiers separated by comma (,) or put this option
# multiple times (only on the command line, not in the configuration file where
# it should appear only once). You can also use "--disable=all" to disable
# everything first and then re-enable specific checks. For example, if you want
# to run only the similarities checker, you can use "--disable=all
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use "--disable=all --enable=classes
# --disable=W".
disable = [
  "prefer-typing-namedtuple",
  "attribute-defined-outside-init",
  "missing-module-docstring",
  "missing-class-docstring",
  "missing-function-docstring",
  "too-few-public-methods",
  "line-too-long",
  "trailing-whitespace",
  "missing-final-newline",
  "trailing-newlines",
  "unnecessary-semicolon",
  "mixed-line-endings",
  "unexpected-line-ending-format",
  "fixme",
  "consider-using-assignment-expr",
  "logging-fstring-interpolation",
  "consider-using-any-or-all"
]

# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
# multiple time (only on the command line, not in the configuration file where it
# should appear only once). See also the "--disable" option for examples.
enable = ["useless-suppression", "use-symbolic-message-instead"]

[tool.pylint.method_args]
# List of qualified names (i.e., library.method) which require a timeout
# parameter e.g. 'requests.api.get,requests.api.post'
timeout-methods = ["requests.api.delete", "requests.api.get", "requests.api.head", "requests.api.options", "requests.api.patch", "requests.api.post", "requests.api.put", "requests.api.request"]

[tool.pylint.miscellaneous]
# List of note tags to take in consideration, separated by a comma.
notes = ["FIXME", "XXX", "TODO"]

# Regular expression of note tags to take in consideration.
# notes-rgx =

[tool.pylint.parameter_documentation]
# Whether to accept totally missing parameter documentation in the docstring of a
# function that has parameters.
accept-no-param-doc = true

# Whether to accept totally missing raises documentation in the docstring of a
# function that raises an exception.
accept-no-raise-doc = true

# Whether to accept totally missing return documentation in the docstring of a
# function that returns a statement.
accept-no-return-doc = true

# Whether to accept totally missing yields documentation in the docstring of a
# generator.
accept-no-yields-doc = true

# If the docstring type cannot be guessed the specified docstring type will be
# used.
default-docstring-type = "default"

[tool.pylint.refactoring]
# Maximum number of nested blocks for function / method body
max-nested-blocks = 5

# Complete name of functions that never returns. When checking for inconsistent-
# return-statements if a never returning function is called then it will be
# considered as an explicit return statement and no message will be printed.
never-returning-functions = ["sys.exit", "argparse.parse_error"]

[tool.pylint.reports]
# Python expression which should return a score less than or equal to 10. You
# have access to the variables 'fatal', 'error', 'warning', 'refactor',
# 'convention', and 'info' which contain the number of messages in each category,
# as well as 'statement' which is the total number of statements analyzed. This
# score is used by the global evaluation report (RP0004).
evaluation = "max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10))"

# Template used to display messages. This is a python new-style format string
# used to format the message information. See doc for all details.
# msg-template =

# Set the output format. Available formats are: text, parseable, colorized, json2
# (improved json format), json (old json format) and msvs (visual studio). You
# can also give a reporter class, e.g. mypackage.mymodule.MyReporterClass.
# output-format =

# Tells whether to display a full report or only the messages.
# reports =

# Activate the evaluation score.
score = true

[tool.pylint.similarities]
# Comments are removed from the similarity computation
ignore-comments = true

# Docstrings are removed from the similarity computation
ignore-docstrings = true

# Imports are removed from the similarity computation
ignore-imports = true

# Signatures are removed from the similarity computation
ignore-signatures = true

# Minimum lines number of a similarity.
min-similarity-lines = 6

[tool.pylint.spelling]
# Limits count of emitted suggestions for spelling mistakes.
max-spelling-suggestions = 2

# Spelling dictionary name. No available dictionaries : You need to install both
# the python package and the system dependency for enchant to work.
# spelling-dict =

# List of comma separated words that should be considered directives if they
# appear at the beginning of a comment and should not be checked.
spelling-ignore-comment-directives = "fmt: on,fmt: off,noqa:,noqa,nosec,mypy:,pragma:,# noinspection"

# List of comma separated words that should not be checked.
# spelling-ignore-words =

# A path to a file that contains the private dictionary; one word per line.
spelling-private-dict-file = ".pyenchant_pylint_custom_dict.txt"

# Tells whether to store unknown words to the private dictionary (see the
# --spelling-private-dict-file option) instead of raising a message.
# spelling-store-unknown-words =

[tool.pylint.typecheck]
# List of decorators that produce context managers, such as
# contextlib.contextmanager. Add to this list to register other decorators that
# produce valid context managers.
contextmanager-decorators = ["contextlib.contextmanager"]

# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E1101 when accessed. Python regular
# expressions are accepted.
generated-members = "REQUEST,acl_users,aq_parent,argparse.Namespace"

# Tells whether missing members accessed in mixin class should be ignored. A
# class is considered mixin if its name matches the mixin-class-rgx option.
# Tells whether to warn about missing members when the owner of the attribute is
# inferred to be None.
ignore-none = true

# This flag controls whether pylint should warn about no-member and similar
# checks whenever an opaque object is returned when inferring. The inference can
# return multiple potential results while evaluating a Python object, but some
# branches might not be evaluated, which results in partial inference. In that
# case, it might be useful to still emit no-member and other checks for the rest
# of the inferred objects.
ignore-on-opaque-inference = true

# List of symbolic message names to ignore for Mixin members.
ignored-checks-for-mixins = ["no-member", "not-async-context-manager", "not-context-manager", "attribute-defined-outside-init"]

# List of class names for which member attributes should not be checked (useful
# for classes with dynamically set attributes). This supports the use of
# qualified names.
ignored-classes = ["SQLObject", "optparse.Values", "thread._local", "_thread._local"]

# Show a hint with possible names when a member name was not found. The aspect of
# finding the hint is based on edit distance.
missing-member-hint = true

# The minimum edit distance a name should have in order to be considered a
# similar match for a missing member name.
missing-member-hint-distance = 1

# The total number of similar names that should be taken in consideration when
# showing a hint for a missing member.
missing-member-max-choices = 1

# Regex pattern to define which classes are considered mixins.
mixin-class-rgx = ".*MixIn"

# List of decorators that change the signature of a decorated function.
# signature-mutators =

[tool.pylint.variables]
# List of additional names supposed to be defined in builtins. Remember that you
# should avoid defining new builtins when possible.
# additional-builtins =

# Tells whether unused global variables should be treated as a violation.
allow-global-unused-variables = true

# List of names allowed to shadow builtins
# allowed-redefined-builtins =

# List of strings which can identify a callback function by name. A callback name
# must start or end with one of those strings.
callbacks = ["cb_", "_cb"]

# A regular expression matching the name of dummy variables (i.e. expected to not
# be used).
dummy-variables-rgx = "_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_"

# Argument names that match this expression will be ignored.
ignored-argument-names = "_.*|^ignored_|^unused_"

# Tells whether we should check for unused import in __init__ files.
# init-import =

# List of qualified module names which can have objects that can redefine
# builtins.
redefining-builtins-modules = ["six.moves", "past.builtins", "future.builtins", "builtins", "io"]