Initial commit

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/push.yml

@@ -0,0 +1,53 @@
+name: build
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+  merge_group:
+    types: [checks_requested]
+  push:
+    # Always run on push to main. The build cache can only be reused
+    # if it was saved by a run from the repository's default branch.
+    # The run result will be identical to that from the merge queue
+    # because the commit is identical, yet we need to perform it to
+    # seed the build cache.
+    branches:
+      - main
+
+jobs:
+  ci:
+    strategy:
+      fail-fast: false
+      matrix:
+        pyVersion: [ '3.10', '3.11', '3.12' ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          cache: 'pip'
+          cache-dependency-path: '**/pyproject.toml'
+          python-version: ${{ matrix.pyVersion }}
+
+      - name: Run unit tests
+        run: |
+          pip install hatch==1.7.0
+          make test
+
+      - name: Publish test coverage
+        uses: codecov/codecov-action@v1
+
+  fmt:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Format all files
+        run: make dev fmt
+
+      - name: Fail on differences
+        run: git diff --exit-code

.gitignore

@@ -1,3 +1,8 @@
+# macos
+
+.DS_Store
+*.DS_Store
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -82,33 +87,6 @@ target/
 profile_default/
 ipython_config.py
 
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/#use-with-ide
-.pdm.toml
-
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
 __pypackages__/
 
@@ -120,13 +98,15 @@ celerybeat.pid
 *.sage.py
 
 # Environments
-.env
+.env.admin
 .venv
+.env.*
 env/
 venv/
 ENV/
 env.bak/
 venv.bak/
+.env
 
 # Spyder project settings
 .spyderproject
@@ -157,4 +137,17 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/
+
+# ruff
+.ruff_cache
+/scratch
+
+# dev files and scratches
+dev/cleanup.py
+
+.databricks
+.vscode
+
+.python-version
+.databricks-login.json

LICENSE

@@ -0,0 +1,69 @@
+                               Databricks License
+                        Copyright (2023) Databricks, Inc.
+
+    Definitions. 
+
+    Agreement: The agreement between Databricks, Inc., and you governing 
+    the use of the Databricks Services, as that term is defined in 
+    the Master Cloud Services Agreement (MCSA) located at 
+    www.databricks.com/legal/mcsa.
+
+    Licensed Materials: The source code, object code, data, and/or other 
+    works to which this license applies. 
+
+    Scope of Use. You may not use the Licensed Materials except in 
+    connection with your use of the Databricks Services pursuant to 
+    the Agreement. Your use of the Licensed Materials must comply at all 
+    times with any restrictions applicable to the Databricks Services, 
+    generally, and must be used in accordance with any applicable 
+    documentation. You may view, use, copy, modify, publish, and/or 
+    distribute the Licensed Materials solely for the purposes of using 
+    the Licensed Materials within or connecting to the Databricks Services.
+    If you do not agree to these terms, you may not view, use, copy, 
+    modify, publish, and/or distribute the Licensed Materials.
+
+    Redistribution. You may redistribute and sublicense the Licensed 
+    Materials so long as all use is in compliance with these terms. 
+    In addition:
+
+        -   You must give any other recipients a copy of this License;
+        -   You must cause any modified files to carry prominent notices 
+            stating that you changed the files; 
+        -   You must retain, in any derivative works that you distribute, 
+            all copyright, patent, trademark, and attribution notices, 
+            excluding those notices that do not pertain to any part of 
+            the derivative works; and
+        -   If a "NOTICE" text file is provided as part of its 
+            distribution, then any derivative works that you distribute 
+            must include a readable copy of the attribution notices 
+            contained within such NOTICE file, excluding those notices 
+            that do not pertain to any part of the derivative works. 
+
+    You may add your own copyright statement to your modifications and may 
+    provide additional license terms and conditions for use, reproduction, 
+    or distribution of your modifications, or for any such derivative works 
+    as a whole, provided your use, reproduction, and distribution of 
+    the Licensed Materials otherwise complies with the conditions stated 
+    in this License.
+
+    Termination. This license terminates automatically upon your breach of 
+    these terms or upon the termination of your Agreement. Additionally, 
+    Databricks may terminate this license at any time on notice. Upon 
+    termination, you must permanently delete the Licensed Materials and 
+    all copies thereof.
+
+    DISCLAIMER; LIMITATION OF LIABILITY. 
+
+    THE LICENSED MATERIALS ARE PROVIDED “AS-IS” AND WITH ALL FAULTS. 
+    DATABRICKS, ON BEHALF OF ITSELF AND ITS LICENSORS, SPECIFICALLY 
+    DISCLAIMS ALL WARRANTIES RELATING TO THE LICENSED MATERIALS, EXPRESS 
+    AND IMPLIED, INCLUDING, WITHOUT LIMITATION, IMPLIED WARRANTIES, 
+    CONDITIONS AND OTHER TERMS OF MERCHANTABILITY, SATISFACTORY QUALITY OR 
+    FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. DATABRICKS AND 
+    ITS LICENSORS TOTAL AGGREGATE LIABILITY RELATING TO OR ARISING OUT OF 
+    YOUR USE OF OR DATABRICKS’ PROVISIONING OF THE LICENSED MATERIALS SHALL 
+    BE LIMITED TO ONE THOUSAND ($1,000) DOLLARS.  IN NO EVENT SHALL 
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 
+    OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 
+    ARISING FROM, OUT OF OR IN CONNECTION WITH THE LICENSED MATERIALS OR 
+    THE USE OR OTHER DEALINGS IN THE LICENSED MATERIALS.

Makefile

@@ -0,0 +1,28 @@
+all: clean lint fmt test coverage
+
+clean:
+	rm -fr .venv clean htmlcov .mypy_cache .pytest_cache .ruff_cache .coverage coverage.xml
+	rm -fr **/*.pyc
+
+.venv/bin/python:
+	pip install hatch
+	hatch env create
+
+dev: .venv/bin/python
+	@hatch run which python
+
+lint:
+	hatch run verify
+
+fmt:
+	hatch run fmt
+
+test:
+	hatch run test
+
+integration:
+	hatch run integration
+
+coverage:
+	hatch run coverage && open htmlcov/index.html
+

NOTICE

@@ -0,0 +1,9 @@
+Copyright (2023) Databricks, Inc.
+
+This Software includes software developed at Databricks (https://www.databricks.com/) and its use is subject to the included LICENSE file.
+
+This Software contains code from the following open source projects, licensed under the Apache 2.0 license:
+
+Databricks SDK for Python - https://github.com/databricks/databricks-sdk-py
+Copyright 2023 Databricks, Inc.  All rights reserved.
+License - https://github.com/databricks/databricks-sdk-py/blob/main/LICENSE

labs.yml

@@ -0,0 +1,10 @@
+---
+name: blueprint
+description: Common libraries for Databricks Labs
+install:
+  script: src/databricks/labs/ucx/install.py
+entrypoint: src/databricks/labs/ucx/cli.py
+min_python: 3.10
+commands:
+  - name: init
+    description: initializes new project

pyproject.toml

@@ -0,0 +1,101 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build]
+sources = ["src"]
+include = ["src"]
+
+[project]
+name = "databricks-labs-blueprint"
+dynamic = ["version"]
+description = 'Common libraries for Databricks Labs'
+readme = "README.md"
+license-files = { paths = ["LICENSE", "NOTICE"] }
+requires-python = ">=3.10.6" # latest available in DBR 13.2
+keywords = ["Databricks"]
+authors = [
+    { name = "Serge Smertin", email = "[email protected]" },
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "License :: Other/Proprietary License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: Implementation :: CPython",
+]
+dependencies = ["databricks-sdk~=0.16.0"]
+
+[project.urls]
+Issues = "https://github.com/databricks/blueprint/issues"
+Source = "https://github.com/databricks/blueprint"
+
+[tool.hatch.version]
+path = "src/databricks/labs/blueprint/__about__.py"
+
+[tool.hatch.envs.default]
+dependencies = [
+    "coverage[toml]>=6.5",
+    "pytest",
+    "pytest-xdist",
+    "pytest-cov>=4.0.0,<5.0.0",
+    "pytest-mock>=3.0.0,<4.0.0",
+    "pytest-timeout",
+    "ruff>=0.0.243",
+    "isort>=2.5.0",
+    "mypy",
+    "types-PyYAML",
+    "types-requests",
+]
+
+python="3.10"
+
+# store virtual env as the child of this folder. Helps VSCode (and PyCharm) to run better
+path = ".venv"
+
+[tool.hatch.envs.default.scripts]
+test        = "pytest -n auto --cov src --cov-report=xml --timeout 30 tests/unit --durations 20"
+coverage    = "pytest -n auto --cov src tests/unit --timeout 30 --cov-report=html --durations 20"
+integration = "pytest -n 10 --cov src tests/integration --durations 20"
+fmt         = ["isort .",
+               "ruff format",
+               "ruff  . --fix",
+               "mypy ."]
+verify      = ["black --check .",
+               "isort . --check-only",
+               "ruff .",
+               "mypy ."]
+
+[tool.isort]
+skip_glob = ["notebooks/*.py"]
+profile = "black"
+
+[tool.pytest.ini_options]
+addopts = "-s -p no:warnings -vv --cache-clear"
+cache_dir = ".venv/pytest-cache"
+
+[tool.black]
+target-version = ["py310"]
+line-length = 120
+skip-string-normalization = true
+
+[tool.ruff]
+cache-dir = ".venv/ruff-cache"
+target-version = "py310"
+line-length = 120
+
+[tool.ruff.isort]
+known-first-party = ["databricks.labs.blueprint"]
+
+[tool.coverage.run]
+branch = true
+parallel = true
+
+[tool.coverage.report]
+omit = ["src/databricks/labs/*", "*/working-copy/*", "*/fresh_wheel_file/*"]
+exclude_lines = [
+    "no cov",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+]

src/databricks/__init__.py

@@ -0,0 +1,3 @@
+# DO NOT ADD ANYTHING ELSE TO THIS FILE FOR COMPATIBILITY WITH OTHER databricks.* PACKAGES
+# SEE https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)

src/databricks/labs/__init__.py

@@ -0,0 +1,3 @@
+# DO NOT ADD ANYTHING ELSE TO THIS FILE FOR COMPATIBILITY WITH OTHER databricks.labs.* PACKAGES
+# SEE https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)

src/databricks/labs/blueprint/__about__.py

@@ -0,0 +1,2 @@
+# DO NOT MODIFY THIS FILE
+__version__ = "0.0.1"