From 86e42a9b84082cf692afb60d47a7688c36ee61c2 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Sun, 1 Nov 2020 23:15:59 +0100 Subject: [PATCH 1/6] [INIT] Full refactor of the OpenUpgrade project --- .copier-answers.yml | 13 + .editorconfig | 20 + .eslintrc.yml | 180 ++++++ .flake8 | 10 + .gitignore | 110 +--- .isort.cfg | 12 + .pre-commit-config.yaml | 131 +++++ .prettierrc.yml | 8 + .pylintrc | 88 +++ .pylintrc-mandatory | 64 ++ .travis.yml | 48 ++ CONTRIBUTING.md | 10 + LICENSE | 4 +- README.md | 37 +- oca_dependencies.txt | 1 + openupgrade_framework/README.rst | 8 + openupgrade_framework/__init__.py | 2 + openupgrade_framework/__manifest__.py | 14 + openupgrade_framework/odoo_patch/__init__.py | 3 + .../odoo_patch/addons/__init__.py | 3 + .../odoo_patch/addons/mrp/__init__.py | 20 + .../addons/point_of_sale/__init__.py | 1 + .../addons/point_of_sale/models/__init__.py | 1 + .../addons/point_of_sale/models/pos_config.py | 21 + .../odoo_patch/addons/stock/__init__.py | 17 + .../odoo_patch/odoo/__init__.py | 10 + openupgrade_framework/odoo_patch/odoo/http.py | 32 + .../odoo_patch/odoo/models.py | 179 ++++++ .../odoo_patch/odoo/modules/__init__.py | 12 + .../odoo_patch/odoo/modules/graph.py | 108 ++++ .../odoo_patch/odoo/modules/loading.py | 556 ++++++++++++++++++ .../odoo_patch/odoo/modules/migration.py | 118 ++++ .../odoo_patch/odoo/modules/registry.py | 58 ++ .../odoo_patch/odoo/service/__init__.py | 4 + .../odoo_patch/odoo/service/server.py | 71 +++ .../odoo_patch/odoo/tools/__init__.py | 2 + .../odoo_patch/odoo/tools/convert.py | 23 + .../odoo_patch/odoo/tools/view_validation.py | 29 + openupgrade_framework/openupgrade/__init__.py | 0 .../openupgrade/openupgrade_loading.py | 318 ++++++++++ .../openupgrade/openupgrade_log.py | 60 ++ openupgrade_framework/readme/CONFIGURE.rst | 7 + openupgrade_framework/readme/CONTRIBUTORS.rst | 2 + openupgrade_framework/readme/DESCRIPTION.rst | 2 + openupgrade_framework/readme/DEVELOP.rst | 60 ++ openupgrade_records/README.rst | 8 + openupgrade_records/__init__.py | 3 + openupgrade_records/__manifest__.py | 26 + openupgrade_records/apriori.py | 25 + openupgrade_records/blacklist.py | 12 + openupgrade_records/compare.py | 483 +++++++++++++++ openupgrade_records/models/__init__.py | 3 + .../models/openupgrade_attribute.py | 18 + .../models/openupgrade_comparison_config.py | 82 +++ .../models/openupgrade_record.py | 99 ++++ openupgrade_records/readme/CONTRIBUTORS.rst | 7 + openupgrade_records/readme/DESCRIPTION.rst | 1 + openupgrade_records/readme/INSTALL.rst | 4 + openupgrade_records/readme/ROADMAP.rst | 3 + openupgrade_records/readme/USAGE.rst | 1 + .../security/ir.model.access.csv | 7 + openupgrade_records/views/menu.xml | 12 + .../views/openupgrade_comparison_config.xml | 73 +++ .../views/openupgrade_record.xml | 84 +++ openupgrade_records/wizards/__init__.py | 3 + .../wizards/openupgrade_analysis_wizard.py | 196 ++++++ .../wizards/openupgrade_analysis_wizard.xml | 35 ++ .../openupgrade_generate_records_wizard.py | 119 ++++ .../openupgrade_generate_records_wizard.xml | 47 ++ .../wizards/openupgrade_install_all_wizard.py | 56 ++ .../openupgrade_install_all_wizard.xml | 49 ++ openupgrade_scripts/README.rst | 8 + openupgrade_scripts/__init__.py | 0 openupgrade_scripts/__manifest__.py | 14 + openupgrade_scripts/readme/CONFIGURE.rst | 10 + openupgrade_scripts/readme/CONTRIBUTORS.rst | 1 + openupgrade_scripts/readme/DESCRIPTION.rst | 1 + .../14.0.1.0/openupgrade_analysis.txt | 5 + .../base/14.0.1.3/openupgrade_analysis.txt | 104 ++++ .../base/14.0.1.3/openupgrade_general_log.txt | 30 + .../14.0.1.0/openupgrade_analysis.txt | 22 + .../web/14.0.1.0/openupgrade_analysis.txt | 15 + .../14.0.1.0/openupgrade_analysis.txt | 23 + .../14.0.0.1/openupgrade_analysis.txt | 14 + requirements.txt | 7 + setup/.setuptools-odoo-make-default-ignore | 2 + setup/README | 2 + .../odoo/addons/openupgrade_framework | 1 + setup/openupgrade_framework/setup.py | 6 + .../odoo/addons/openupgrade_records | 1 + setup/openupgrade_records/setup.py | 6 + .../odoo/addons/openupgrade_scripts | 1 + setup/openupgrade_scripts/setup.py | 6 + 93 files changed, 4127 insertions(+), 85 deletions(-) create mode 100644 .copier-answers.yml create mode 100644 .editorconfig create mode 100644 .eslintrc.yml create mode 100644 .flake8 create mode 100644 .isort.cfg create mode 100644 .pre-commit-config.yaml create mode 100644 .prettierrc.yml create mode 100644 .pylintrc create mode 100644 .pylintrc-mandatory create mode 100644 .travis.yml create mode 100644 CONTRIBUTING.md create mode 100644 oca_dependencies.txt create mode 100644 openupgrade_framework/README.rst create mode 100644 openupgrade_framework/__init__.py create mode 100644 openupgrade_framework/__manifest__.py create mode 100644 openupgrade_framework/odoo_patch/__init__.py create mode 100644 openupgrade_framework/odoo_patch/addons/__init__.py create mode 100644 openupgrade_framework/odoo_patch/addons/mrp/__init__.py create mode 100644 openupgrade_framework/odoo_patch/addons/point_of_sale/__init__.py create mode 100644 openupgrade_framework/odoo_patch/addons/point_of_sale/models/__init__.py create mode 100644 openupgrade_framework/odoo_patch/addons/point_of_sale/models/pos_config.py create mode 100644 openupgrade_framework/odoo_patch/addons/stock/__init__.py create mode 100644 openupgrade_framework/odoo_patch/odoo/__init__.py create mode 100644 openupgrade_framework/odoo_patch/odoo/http.py create mode 100644 openupgrade_framework/odoo_patch/odoo/models.py create mode 100644 openupgrade_framework/odoo_patch/odoo/modules/__init__.py create mode 100644 openupgrade_framework/odoo_patch/odoo/modules/graph.py create mode 100644 openupgrade_framework/odoo_patch/odoo/modules/loading.py create mode 100644 openupgrade_framework/odoo_patch/odoo/modules/migration.py create mode 100644 openupgrade_framework/odoo_patch/odoo/modules/registry.py create mode 100644 openupgrade_framework/odoo_patch/odoo/service/__init__.py create mode 100644 openupgrade_framework/odoo_patch/odoo/service/server.py create mode 100644 openupgrade_framework/odoo_patch/odoo/tools/__init__.py create mode 100644 openupgrade_framework/odoo_patch/odoo/tools/convert.py create mode 100644 openupgrade_framework/odoo_patch/odoo/tools/view_validation.py create mode 100644 openupgrade_framework/openupgrade/__init__.py create mode 100644 openupgrade_framework/openupgrade/openupgrade_loading.py create mode 100644 openupgrade_framework/openupgrade/openupgrade_log.py create mode 100644 openupgrade_framework/readme/CONFIGURE.rst create mode 100644 openupgrade_framework/readme/CONTRIBUTORS.rst create mode 100644 openupgrade_framework/readme/DESCRIPTION.rst create mode 100644 openupgrade_framework/readme/DEVELOP.rst create mode 100644 openupgrade_records/README.rst create mode 100644 openupgrade_records/__init__.py create mode 100644 openupgrade_records/__manifest__.py create mode 100644 openupgrade_records/apriori.py create mode 100644 openupgrade_records/blacklist.py create mode 100644 openupgrade_records/compare.py create mode 100644 openupgrade_records/models/__init__.py create mode 100644 openupgrade_records/models/openupgrade_attribute.py create mode 100644 openupgrade_records/models/openupgrade_comparison_config.py create mode 100644 openupgrade_records/models/openupgrade_record.py create mode 100644 openupgrade_records/readme/CONTRIBUTORS.rst create mode 100644 openupgrade_records/readme/DESCRIPTION.rst create mode 100644 openupgrade_records/readme/INSTALL.rst create mode 100644 openupgrade_records/readme/ROADMAP.rst create mode 100644 openupgrade_records/readme/USAGE.rst create mode 100644 openupgrade_records/security/ir.model.access.csv create mode 100644 openupgrade_records/views/menu.xml create mode 100644 openupgrade_records/views/openupgrade_comparison_config.xml create mode 100644 openupgrade_records/views/openupgrade_record.xml create mode 100644 openupgrade_records/wizards/__init__.py create mode 100644 openupgrade_records/wizards/openupgrade_analysis_wizard.py create mode 100644 openupgrade_records/wizards/openupgrade_analysis_wizard.xml create mode 100644 openupgrade_records/wizards/openupgrade_generate_records_wizard.py create mode 100644 openupgrade_records/wizards/openupgrade_generate_records_wizard.xml create mode 100644 openupgrade_records/wizards/openupgrade_install_all_wizard.py create mode 100644 openupgrade_records/wizards/openupgrade_install_all_wizard.xml create mode 100644 openupgrade_scripts/README.rst create mode 100644 openupgrade_scripts/__init__.py create mode 100644 openupgrade_scripts/__manifest__.py create mode 100644 openupgrade_scripts/readme/CONFIGURE.rst create mode 100644 openupgrade_scripts/readme/CONTRIBUTORS.rst create mode 100644 openupgrade_scripts/readme/DESCRIPTION.rst create mode 100644 openupgrade_scripts/scripts/auth_totp/14.0.1.0/openupgrade_analysis.txt create mode 100644 openupgrade_scripts/scripts/base/14.0.1.3/openupgrade_analysis.txt create mode 100644 openupgrade_scripts/scripts/base/14.0.1.3/openupgrade_general_log.txt create mode 100644 openupgrade_scripts/scripts/base_import/14.0.1.0/openupgrade_analysis.txt create mode 100644 openupgrade_scripts/scripts/web/14.0.1.0/openupgrade_analysis.txt create mode 100644 openupgrade_scripts/scripts/web_editor/14.0.1.0/openupgrade_analysis.txt create mode 100644 openupgrade_scripts/scripts/web_tour/14.0.0.1/openupgrade_analysis.txt create mode 100644 requirements.txt create mode 100644 setup/.setuptools-odoo-make-default-ignore create mode 100644 setup/README create mode 120000 setup/openupgrade_framework/odoo/addons/openupgrade_framework create mode 100644 setup/openupgrade_framework/setup.py create mode 120000 setup/openupgrade_records/odoo/addons/openupgrade_records create mode 100644 setup/openupgrade_records/setup.py create mode 120000 setup/openupgrade_scripts/odoo/addons/openupgrade_scripts create mode 100644 setup/openupgrade_scripts/setup.py diff --git a/.copier-answers.yml b/.copier-answers.yml new file mode 100644 index 000000000000..c98369f2a57d --- /dev/null +++ b/.copier-answers.yml @@ -0,0 +1,13 @@ +# Do NOT update manually; changes here will be overwritten by Copier +_commit: v1.0.3 +_src_path: https://github.com/OCA/oca-addons-repo-template.git +dependency_installation_mode: OCA +generate_requirements_txt: false +include_wkhtmltopdf: false +odoo_version: 14.0 +rebel_module_groups: [] +repo_description: null +repo_name: Tools to upgrade Odoo instances from a major version to another +repo_slug: openupgrade +travis_apt_packages: [] +travis_apt_sources: [] diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000000..bfd7ac53df9f --- /dev/null +++ b/.editorconfig @@ -0,0 +1,20 @@ +# Configuration for known file extensions +[*.{css,js,json,less,md,py,rst,sass,scss,xml,yaml,yml}] +charset = utf-8 +end_of_line = lf +indent_size = 4 +indent_style = space +insert_final_newline = true +trim_trailing_whitespace = true + +[*.{json,yml,yaml,rst,md}] +indent_size = 2 + +# Do not configure editor for libs and autogenerated content +[{*/static/{lib,src/lib}/**,*/static/description/index.html,*/readme/../README.rst}] +charset = unset +end_of_line = unset +indent_size = unset +indent_style = unset +insert_final_newline = false +trim_trailing_whitespace = false diff --git a/.eslintrc.yml b/.eslintrc.yml new file mode 100644 index 000000000000..88f2881b4c12 --- /dev/null +++ b/.eslintrc.yml @@ -0,0 +1,180 @@ +env: + browser: true + +# See https://github.com/OCA/odoo-community.org/issues/37#issuecomment-470686449 +parserOptions: + ecmaVersion: 2017 + +# Globals available in Odoo that shouldn't produce errorings +globals: + _: readonly + $: readonly + fuzzy: readonly + jQuery: readonly + moment: readonly + odoo: readonly + openerp: readonly + Promise: readonly + +# Styling is handled by Prettier, so we only need to enable AST rules; +# see https://github.com/OCA/maintainer-quality-tools/pull/618#issuecomment-558576890 +rules: + accessor-pairs: warn + array-callback-return: warn + callback-return: warn + capitalized-comments: + - warn + - always + - ignoreConsecutiveComments: true + ignoreInlineComments: true + complexity: + - warn + - 15 + constructor-super: warn + dot-notation: warn + eqeqeq: warn + global-require: warn + handle-callback-err: warn + id-blacklist: warn + id-match: warn + init-declarations: error + max-depth: warn + max-nested-callbacks: warn + max-statements-per-line: warn + no-alert: warn + no-array-constructor: warn + no-caller: warn + no-case-declarations: warn + no-class-assign: warn + no-cond-assign: error + no-const-assign: error + no-constant-condition: warn + no-control-regex: warn + no-debugger: error + no-delete-var: warn + no-div-regex: warn + no-dupe-args: error + no-dupe-class-members: error + no-dupe-keys: error + no-duplicate-case: error + no-duplicate-imports: error + no-else-return: warn + no-empty-character-class: warn + no-empty-function: error + no-empty-pattern: error + no-empty: warn + no-eq-null: error + no-eval: error + no-ex-assign: error + no-extend-native: warn + no-extra-bind: warn + no-extra-boolean-cast: warn + no-extra-label: warn + no-fallthrough: warn + no-func-assign: error + no-global-assign: error + no-implicit-coercion: + - warn + - allow: ["~"] + no-implicit-globals: warn + no-implied-eval: warn + no-inline-comments: warn + no-inner-declarations: warn + no-invalid-regexp: warn + no-irregular-whitespace: warn + no-iterator: warn + no-label-var: warn + no-labels: warn + no-lone-blocks: warn + no-lonely-if: error + no-mixed-requires: error + no-multi-str: warn + no-native-reassign: error + no-negated-condition: warn + no-negated-in-lhs: error + no-new-func: warn + no-new-object: warn + no-new-require: warn + no-new-symbol: warn + no-new-wrappers: warn + no-new: warn + no-obj-calls: warn + no-octal-escape: warn + no-octal: warn + no-param-reassign: warn + no-path-concat: warn + no-process-env: warn + no-process-exit: warn + no-proto: warn + no-prototype-builtins: warn + no-redeclare: warn + no-regex-spaces: warn + no-restricted-globals: warn + no-restricted-imports: warn + no-restricted-modules: warn + no-restricted-syntax: warn + no-return-assign: error + no-script-url: warn + no-self-assign: warn + no-self-compare: warn + no-sequences: warn + no-shadow-restricted-names: warn + no-shadow: warn + no-sparse-arrays: warn + no-sync: warn + no-this-before-super: warn + no-throw-literal: warn + no-undef-init: warn + no-undef: error + no-unmodified-loop-condition: warn + no-unneeded-ternary: error + no-unreachable: error + no-unsafe-finally: error + no-unused-expressions: error + no-unused-labels: error + no-unused-vars: error + no-use-before-define: error + no-useless-call: warn + no-useless-computed-key: warn + no-useless-concat: warn + no-useless-constructor: warn + no-useless-escape: warn + no-useless-rename: warn + no-void: warn + no-with: warn + operator-assignment: [error, always] + prefer-const: warn + radix: warn + require-yield: warn + sort-imports: warn + spaced-comment: [error, always] + strict: [error, function] + use-isnan: error + valid-jsdoc: + - warn + - prefer: + arg: param + argument: param + augments: extends + constructor: class + exception: throws + func: function + method: function + prop: property + return: returns + virtual: abstract + yield: yields + preferType: + array: Array + bool: Boolean + boolean: Boolean + number: Number + object: Object + str: String + string: String + requireParamDescription: false + requireReturn: false + requireReturnDescription: false + requireReturnType: false + valid-typeof: warn + yoda: warn diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000000..44ed868f7ff2 --- /dev/null +++ b/.flake8 @@ -0,0 +1,10 @@ +[flake8] +max-line-length = 80 +max-complexity = 16 +# B = bugbear +# B9 = bugbear opinionated (incl line length) +select = C,E,F,W,B,B9 +# E203: whitespace before ':' (black behaviour) +# E501: flake8 line length (covered by bugbear B950) +# W503: line break before binary operator (black behaviour) +ignore = E203,E501,W503 diff --git a/.gitignore b/.gitignore index b6e47617de11..818770fb1bdc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,37 +1,29 @@ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] -*$py.class +/.venv +/.pytest_cache # C extensions *.so # Distribution / packaging .Python +env/ +bin/ build/ develop-eggs/ dist/ -downloads/ eggs/ -.eggs/ lib/ lib64/ parts/ sdist/ var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ *.egg-info/ .installed.cfg *.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec +*.eggs # Installer logs pip-log.txt @@ -40,90 +32,44 @@ pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ -.nox/ .coverage -.coverage.* .cache nosetests.xml coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ # Translations *.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ -# PyBuilder -target/ +# Pycharm +.idea -# Jupyter Notebook -.ipynb_checkpoints +# Eclipse +.settings -# IPython -profile_default/ -ipython_config.py +# Visual Studio cache/options directory +.vs/ +.vscode -# pyenv -.python-version +# OSX Files +.DS_Store -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ +# Django stuff: +*.log -# Spyder project settings -.spyderproject -.spyproject +# Mr Developer +.mr.developer.cfg +.project +.pydevproject -# Rope project settings +# Rope .ropeproject -# mkdocs documentation -/site +# Sphinx documentation +docs/_build/ -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json +# Backup files +*~ +*.swp -# Pyre type checker -.pyre/ +# OCA rules +!static/lib/ diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 000000000000..7683badfa449 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,12 @@ +[settings] +; see https://github.com/psf/black +multi_line_output=3 +include_trailing_comma=True +force_grid_wrap=0 +combine_as_imports=True +use_parentheses=True +line_length=88 +known_odoo=odoo +known_odoo_addons=odoo.addons +sections=FUTURE,STDLIB,THIRDPARTY,ODOO,ODOO_ADDONS,FIRSTPARTY,LOCALFOLDER +default_section=THIRDPARTY diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000000..d842ef6909e7 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,131 @@ +exclude: | + (?x) + # NOT INSTALLABLE ADDONS + ^openupgrade_framework/| + # END NOT INSTALLABLE ADDONS + # Files and folders generated by bots, to avoid loops + ^setup/|/static/description/index\.html$| + # Maybe reactivate this when all README files include prettier ignore tags? + ^README\.md$| + # Library files can have extraneous formatting (even minimized) + /static/(src/)?lib/| + # Repos using Sphinx to generate docs don't need prettying + ^docs/_templates/.*\.html$| + # You don't usually want a bot to modify your legal texts + (LICENSE.*|COPYING.*) +default_language_version: + python: python3 + node: "14.13.0" +repos: + - repo: local + hooks: + # These files are most likely copier diff rejection junks; if found, + # review them manually, fix the problem (if needed) and remove them + - id: forbidden-files + name: forbidden files + entry: found forbidden files; remove them + language: fail + files: "\\.rej$" + - repo: https://github.com/oca/maintainer-tools + rev: 1b5c7ad + hooks: + # update the NOT INSTALLABLE ADDONS section above + - id: oca-update-pre-commit-excluded-addons + - id: oca-fix-manifest-website + args: ["https://github.com/OCA/openupgrade"] + - repo: https://github.com/myint/autoflake + rev: v1.4 + hooks: + - id: autoflake + args: ["-i", "--ignore-init-module-imports"] + - repo: https://github.com/psf/black + rev: 20.8b1 + hooks: + - id: black + exclude: ^openupgrade_framework/odoo_patch/ + - repo: https://github.com/prettier/pre-commit + rev: v2.1.2 + hooks: + - id: prettier + name: prettier + plugin-xml + additional_dependencies: + - "prettier@2.1.2" + - "@prettier/plugin-xml@0.12.0" + args: + - --plugin=@prettier/plugin-xml + exclude: ^openupgrade_framework/odoo_patch/ + - repo: https://github.com/pre-commit/mirrors-eslint + rev: v7.8.1 + hooks: + - id: eslint + verbose: true + args: + - --color + - --fix + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + # exclude autogenerated files + exclude: /README\.rst$|\.pot?$ + - id: end-of-file-fixer + # exclude autogenerated files + exclude: /README\.rst$|\.pot?$|^openupgrade_scripts/script.*.txt + - id: debug-statements + - id: fix-encoding-pragma + args: ["--remove"] + - id: check-case-conflict + - id: check-docstring-first + - id: check-executables-have-shebangs + - id: check-merge-conflict + # exclude files where underlines are not distinguishable from merge conflicts + exclude: /README\.rst$|^docs/.*\.rst$ + - id: check-symlinks + - id: check-xml + - id: mixed-line-ending + args: ["--fix=lf"] + - repo: https://github.com/asottile/pyupgrade + rev: v2.7.2 + hooks: + - id: pyupgrade + exclude: ^openupgrade_framework/odoo_patch/ + - repo: https://github.com/PyCQA/isort + rev: 5.5.1 + hooks: + - id: isort + name: isort except __init__.py + args: + - --settings=. + exclude: /__init__\.py$|^openupgrade_framework/odoo_patch/ + - repo: https://github.com/acsone/setuptools-odoo + rev: 2.6.0 + hooks: + - id: setuptools-odoo-make-default + - repo: https://gitlab.com/PyCQA/flake8 + rev: 3.8.3 + hooks: + - id: flake8 + name: flake8 except __init__.py + exclude: /__init__\.py$|^openupgrade_framework/odoo_patch/ + additional_dependencies: ["flake8-bugbear==20.1.4"] + - id: flake8 + name: flake8 only __init__.py + args: ["--extend-ignore=F401"] # ignore unused imports in __init__.py + files: /__init__\.py$ + additional_dependencies: ["flake8-bugbear==20.1.4"] + - repo: https://github.com/PyCQA/pylint + rev: pylint-2.5.3 + hooks: + - id: pylint + name: pylint with optional checks + args: + - --rcfile=.pylintrc + - --exit-zero + verbose: true + additional_dependencies: &pylint_deps + - pylint-odoo==3.5.0 + - id: pylint + name: pylint with mandatory checks + args: + - --rcfile=.pylintrc-mandatory + additional_dependencies: *pylint_deps diff --git a/.prettierrc.yml b/.prettierrc.yml new file mode 100644 index 000000000000..5b6d4b361ace --- /dev/null +++ b/.prettierrc.yml @@ -0,0 +1,8 @@ +# Defaults for all prettier-supported languages. +# Prettier will complete this with settings from .editorconfig file. +bracketSpacing: false +printWidth: 88 +proseWrap: always +semi: true +trailingComma: "es5" +xmlWhitespaceSensitivity: "strict" diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 000000000000..e33958833edc --- /dev/null +++ b/.pylintrc @@ -0,0 +1,88 @@ +[MASTER] +load-plugins=pylint_odoo +score=n + +[ODOOLINT] +readme_template_url="https://github.com/OCA/maintainer-tools/blob/master/template/module/README.rst" +manifest_required_authors=Odoo Community Association (OCA) +manifest_required_keys=license +manifest_deprecated_keys=description,active +license_allowed=AGPL-3,GPL-2,GPL-2 or any later version,GPL-3,GPL-3 or any later version,LGPL-3 +valid_odoo_versions=14.0 + +[MESSAGES CONTROL] +disable=all + +# This .pylintrc contains optional AND mandatory checks and is meant to be +# loaded in an IDE to have it check everything, in the hope this will make +# optional checks more visible to contributors who otherwise never look at a +# green travis to see optional checks that failed. +# .pylintrc-mandatory containing only mandatory checks is used the pre-commit +# config as a blocking check. + +enable=anomalous-backslash-in-string, + api-one-deprecated, + api-one-multi-together, + assignment-from-none, + attribute-deprecated, + class-camelcase, + dangerous-default-value, + dangerous-view-replace-wo-priority, + development-status-allowed, + duplicate-id-csv, + duplicate-key, + duplicate-xml-fields, + duplicate-xml-record-id, + eval-referenced, + eval-used, + incoherent-interpreter-exec-perm, + license-allowed, + manifest-author-string, + manifest-deprecated-key, + manifest-required-author, + manifest-required-key, + manifest-version-format, + method-compute, + method-inverse, + method-required-super, + method-search, + openerp-exception-warning, + pointless-statement, + pointless-string-statement, + print-used, + redundant-keyword-arg, + redundant-modulename-xml, + reimported, + relative-import, + return-in-init, + rst-syntax-error, + sql-injection, + too-few-format-args, + translation-field, + translation-required, + unreachable, + use-vim-comment, + wrong-tabs-instead-of-spaces, + xml-syntax-error, + # messages that do not cause the lint step to fail + consider-merging-classes-inherited, + create-user-wo-reset-password, + dangerous-filter-wo-user, + deprecated-module, + file-not-used, + invalid-commit, + missing-manifest-dependency, + missing-newline-extrafiles, + missing-readme, + no-utf8-coding-comment, + odoo-addons-relative-import, + old-api7-method-defined, + redefined-builtin, + too-complex, + unnecessary-utf8-coding-comment + + +[REPORTS] +msg-template={path}:{line}: [{msg_id}({symbol}), {obj}] {msg} +output-format=colorized +reports=no diff --git a/.pylintrc-mandatory b/.pylintrc-mandatory new file mode 100644 index 000000000000..08bca3cefef2 --- /dev/null +++ b/.pylintrc-mandatory @@ -0,0 +1,64 @@ +[MASTER] +load-plugins=pylint_odoo +score=n + +[ODOOLINT] +readme_template_url="https://github.com/OCA/maintainer-tools/blob/master/template/module/README.rst" +manifest_required_authors=Odoo Community Association (OCA) +manifest_required_keys=license +manifest_deprecated_keys=description,active +license_allowed=AGPL-3,GPL-2,GPL-2 or any later version,GPL-3,GPL-3 or any later version,LGPL-3 +valid_odoo_versions=14.0 + +[MESSAGES CONTROL] +disable=all + +enable=anomalous-backslash-in-string, + api-one-deprecated, + api-one-multi-together, + assignment-from-none, + attribute-deprecated, + class-camelcase, + dangerous-default-value, + dangerous-view-replace-wo-priority, + development-status-allowed, + duplicate-id-csv, + duplicate-key, + duplicate-xml-fields, + duplicate-xml-record-id, + eval-referenced, + eval-used, + incoherent-interpreter-exec-perm, + license-allowed, + manifest-author-string, + manifest-deprecated-key, + manifest-required-author, + manifest-required-key, + manifest-version-format, + method-compute, + method-inverse, + method-required-super, + method-search, + openerp-exception-warning, + pointless-statement, + pointless-string-statement, + print-used, + redundant-keyword-arg, + redundant-modulename-xml, + reimported, + relative-import, + return-in-init, + rst-syntax-error, + sql-injection, + too-few-format-args, + translation-field, + translation-required, + unreachable, + use-vim-comment, + wrong-tabs-instead-of-spaces, + xml-syntax-error + +[REPORTS] +msg-template={path}:{line}: [{msg_id}({symbol}), {obj}] {msg} +output-format=colorized +reports=no diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000000..c0f6df9b5e06 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,48 @@ +language: python +cache: + directories: + - $HOME/.cache/pip + - $HOME/.cache/pre-commit + +python: + - "3.6" + +addons: + postgresql: "9.6" + apt: + packages: + - expect-dev # provides unbuffer utility + +stages: + - linting + - test + +jobs: + include: + - stage: linting + name: "pre-commit" + install: pip install pre-commit + script: pre-commit run --all --show-diff-on-failure --verbose --color always + after_success: + before_install: + - stage: test + env: + - TESTS=1 ODOO_REPO="odoo/odoo" MAKEPOT="1" + - stage: test + env: + - TESTS=1 ODOO_REPO="OCA/OCB" +env: + global: + - VERSION="14.0" TESTS="0" LINT_CHECK="0" MAKEPOT="0" + +install: + - git clone --depth=1 https://github.com/OCA/maintainer-quality-tools.git + ${HOME}/maintainer-quality-tools + - export PATH=${HOME}/maintainer-quality-tools/travis:${PATH} + - travis_install_nightly + +script: + - travis_run_tests + +after_success: + - travis_after_tests_success diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000000..9ac71fee42a5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,10 @@ +# OCA Guidelines + +Please follow the official guide from the +[OCA Guidelines page](https://odoo-community.org/page/contributing). + +## Project Specific Guidelines + + + +This project does not have specific coding guidelines. diff --git a/LICENSE b/LICENSE index 0ad25db4bd1d..be3f7b28e564 100644 --- a/LICENSE +++ b/LICENSE @@ -633,8 +633,8 @@ the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, diff --git a/README.md b/README.md index f780aba64cd8..55eb763603ee 100644 --- a/README.md +++ b/README.md @@ -1 +1,36 @@ -# openupgrade-framework \ No newline at end of file +[![Runbot Status](https://runbot.odoo-community.org/runbot/badge/flat//14.0.svg)](https://runbot.odoo-community.org/runbot/repo/github.aaakk.us.kg-oca-openupgrade-) +[![Build Status](https://travis-ci.com/OCA/openupgrade.svg?branch=14.0)](https://travis-ci.com/OCA/openupgrade) +[![codecov](https://codecov.io/gh/OCA/openupgrade/branch/14.0/graph/badge.svg)](https://codecov.io/gh/OCA/openupgrade) +[![Translation Status](https://translation.odoo-community.org/widgets/openupgrade-14-0/-/svg-badge.svg)](https://translation.odoo-community.org/engage/openupgrade-14-0/?utm_source=widget) + + + +# Tools to upgrade Odoo instances from a major version to another + +None + + + + + +[//]: # (addons) + +This part will be replaced when running the oca-gen-addons-table script from OCA/maintainer-tools. + +[//]: # (end addons) + + + +## Licenses + +This repository is licensed under [AGPL-3.0](LICENSE). + +However, each module can have a totally different license, as long as they adhere to OCA +policy. Consult each module's `__manifest__.py` file, which contains a `license` key +that explains its license. + +---- + +OCA, or the [Odoo Community Association](http://odoo-community.org/), is a nonprofit +organization whose mission is to support the collaborative development of Odoo features +and promote its widespread use. diff --git a/oca_dependencies.txt b/oca_dependencies.txt new file mode 100644 index 000000000000..ca3c726ba20d --- /dev/null +++ b/oca_dependencies.txt @@ -0,0 +1 @@ +# See https://github.com/OCA/odoo-community.org/blob/master/website/Contribution/CONTRIBUTING.rst#oca_dependencies-txt diff --git a/openupgrade_framework/README.rst b/openupgrade_framework/README.rst new file mode 100644 index 000000000000..3ed54188c923 --- /dev/null +++ b/openupgrade_framework/README.rst @@ -0,0 +1,8 @@ +===================== +Openupgrade Framework +===================== + +.. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! diff --git a/openupgrade_framework/__init__.py b/openupgrade_framework/__init__.py new file mode 100644 index 000000000000..94ba6ffa9f2f --- /dev/null +++ b/openupgrade_framework/__init__.py @@ -0,0 +1,2 @@ +from . import odoo_patch +from . import openupgrade diff --git a/openupgrade_framework/__manifest__.py b/openupgrade_framework/__manifest__.py new file mode 100644 index 000000000000..fd016de4ca7a --- /dev/null +++ b/openupgrade_framework/__manifest__.py @@ -0,0 +1,14 @@ +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +{ + "name": "Openupgrade Framework", + "summary": """Module to integrate in the server_wide_modules + option to make upgrades between two major revisions.""", + "author": "Odoo Community Association (OCA)," " Therp BV, Opener B.V., GRAP", + "website": "https://github.com/OCA/openupgrade", + "category": "Migration", + "version": "14.0.1.0.0", + "license": "AGPL-3", + "depends": ["base"], + "installable": False, +} diff --git a/openupgrade_framework/odoo_patch/__init__.py b/openupgrade_framework/odoo_patch/__init__.py new file mode 100644 index 000000000000..1fd6e167cd12 --- /dev/null +++ b/openupgrade_framework/odoo_patch/__init__.py @@ -0,0 +1,3 @@ +from . import odoo +from . import addons + diff --git a/openupgrade_framework/odoo_patch/addons/__init__.py b/openupgrade_framework/odoo_patch/addons/__init__.py new file mode 100644 index 000000000000..e5aa886bacb6 --- /dev/null +++ b/openupgrade_framework/odoo_patch/addons/__init__.py @@ -0,0 +1,3 @@ +from . import mrp +from . import stock +from . import point_of_sale diff --git a/openupgrade_framework/odoo_patch/addons/mrp/__init__.py b/openupgrade_framework/odoo_patch/addons/mrp/__init__.py new file mode 100644 index 000000000000..f7b8b869472b --- /dev/null +++ b/openupgrade_framework/odoo_patch/addons/mrp/__init__.py @@ -0,0 +1,20 @@ +from odoo.addons import mrp + + +def _pre_init_mrp(cr): + """ Allow installing MRP in databases with large stock.move table (>1M records) + - Creating the computed+stored field stock_move.is_done is terribly slow with the ORM and + leads to "Out of Memory" crashes + """ + # + # don't try to add 'is_done' column, because it will fail + # when executing the generation of records, in the openupgrade_records + # module. + # cr.execute("""ALTER TABLE "stock_move" ADD COLUMN "is_done" bool;""") + # cr.execute("""UPDATE stock_move + # SET is_done=COALESCE(state in ('done', 'cancel'), FALSE);""") + pass + # + + +mrp._pre_init_mrp = _pre_init_mrp diff --git a/openupgrade_framework/odoo_patch/addons/point_of_sale/__init__.py b/openupgrade_framework/odoo_patch/addons/point_of_sale/__init__.py new file mode 100644 index 000000000000..0650744f6bc6 --- /dev/null +++ b/openupgrade_framework/odoo_patch/addons/point_of_sale/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/openupgrade_framework/odoo_patch/addons/point_of_sale/models/__init__.py b/openupgrade_framework/odoo_patch/addons/point_of_sale/models/__init__.py new file mode 100644 index 000000000000..db8634ade1f7 --- /dev/null +++ b/openupgrade_framework/odoo_patch/addons/point_of_sale/models/__init__.py @@ -0,0 +1 @@ +from . import pos_config diff --git a/openupgrade_framework/odoo_patch/addons/point_of_sale/models/pos_config.py b/openupgrade_framework/odoo_patch/addons/point_of_sale/models/pos_config.py new file mode 100644 index 000000000000..ac0f5dc5a49e --- /dev/null +++ b/openupgrade_framework/odoo_patch/addons/point_of_sale/models/pos_config.py @@ -0,0 +1,21 @@ +from odoo import api +from odoo.addons.point_of_sale.models.pos_config import PosConfig + +if True: + + @api.model + def post_install_pos_localisation(self, companies=False): + # + # don't try to setup_defaults, because it will fail + # when executing the generation of records, in the openupgrade_records + # module. + # self = self.sudo() + # if not companies: + # companies = self.env['res.company'].search([]) + # for company in companies.filtered('chart_template_id'): + # pos_configs = self.search([('company_id', '=', company.id)]) + # pos_configs.setup_defaults(company) + pass + # + +PosConfig.post_install_pos_localisation = post_install_pos_localisation diff --git a/openupgrade_framework/odoo_patch/addons/stock/__init__.py b/openupgrade_framework/odoo_patch/addons/stock/__init__.py new file mode 100644 index 000000000000..b66d7f484cb1 --- /dev/null +++ b/openupgrade_framework/odoo_patch/addons/stock/__init__.py @@ -0,0 +1,17 @@ +from odoo.addons import stock + + +def pre_init_hook(cr): + # + # don't uninstall data as this breaks the analysis + # Origin of this code is https://github.com/odoo/odoo/issues/22243 + # env = api.Environment(cr, SUPERUSER_ID, {}) + # env['ir.model.data'].search([ + # ('model', 'like', '%stock%'), + # ('module', '=', 'stock') + # ]).unlink() + pass + # + + +stock.pre_init_hook = pre_init_hook diff --git a/openupgrade_framework/odoo_patch/odoo/__init__.py b/openupgrade_framework/odoo_patch/odoo/__init__.py new file mode 100644 index 000000000000..f5065ae34593 --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/__init__.py @@ -0,0 +1,10 @@ +from . import modules +from . import service +from . import tools + +# Nothing todo the function, the function check_security didn't changed +from . import http + +# adapted to V14 +# TODO, OpenUpgrade maintainers : check if it's OK +from . import models diff --git a/openupgrade_framework/odoo_patch/odoo/http.py b/openupgrade_framework/odoo_patch/odoo/http.py new file mode 100644 index 000000000000..e11c558fb905 --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/http.py @@ -0,0 +1,32 @@ +# flake8: noqa +# pylint: skip-file + +import odoo +from odoo.service import security +from odoo.http import SessionExpiredException, request, OpenERPSession + +if True: + def _check_security(self): + """ + Check the current authentication parameters to know if those are still + valid. This method should be called at each request. If the + authentication fails, a :exc:`SessionExpiredException` is raised. + """ + if not self.db or not self.uid: + raise SessionExpiredException("Session expired") + # We create our own environment instead of the request's one. + # to avoid creating it without the uid since request.uid isn't set yet + env = odoo.api.Environment(request.cr, self.uid, self.context) + # here we check if the session is still valid + if not security.check_session(self, env): + # + # When asking openupgrade_records to generate records + # over jsonrpc, a query on res_users in the call above locks this + # table for the sql operations that are triggered by the + # reinstallation of the base module + env.cr.rollback() + # + raise SessionExpiredException("Session expired") + + +OpenERPSession.check_security = _check_security diff --git a/openupgrade_framework/odoo_patch/odoo/models.py b/openupgrade_framework/odoo_patch/odoo/models.py new file mode 100644 index 000000000000..ee09595fb103 --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/models.py @@ -0,0 +1,179 @@ +# flake8: noqa +# pylint: skip-file + +import odoo +import psycopg2 +from odoo import _ +from odoo.models import fix_import_export_id_paths, BaseModel, _logger +from odoo.addons.openupgrade_framework.openupgrade import openupgrade_log + + +if True: + def _load(self, fields, data): + """ + Attempts to load the data matrix, and returns a list of ids (or + ``False`` if there was an error and no id could be generated) and a + list of messages. + + The ids are those of the records created and saved (in database), in + the same order they were extracted from the file. They can be passed + directly to :meth:`~read` + + :param fields: list of fields to import, at the same index as the corresponding data + :type fields: list(str) + :param data: row-major matrix of data to import + :type data: list(list(str)) + :returns: {ids: list(int)|False, messages: [Message][, lastrow: int]} + """ + self.flush() + + # determine values of mode, current_module and noupdate + mode = self._context.get('mode', 'init') + current_module = self._context.get('module', '__import__') + noupdate = self._context.get('noupdate', False) + # add current module in context for the conversion of xml ids + self = self.with_context(_import_current_module=current_module) + + cr = self._cr + cr.execute('SAVEPOINT model_load') + + fields = [fix_import_export_id_paths(f) for f in fields] + fg = self.fields_get() + + ids = [] + messages = [] + ModelData = self.env['ir.model.data'] + + # list of (xid, vals, info) for records to be created in batch + batch = [] + batch_xml_ids = set() + # models in which we may have created / modified data, therefore might + # require flushing in order to name_search: the root model and any + # o2m + creatable_models = {self._name} + for field_path in fields: + if field_path[0] in (None, 'id', '.id'): + continue + model_fields = self._fields + if isinstance(model_fields[field_path[0]], odoo.fields.Many2one): + # this only applies for toplevel m2o (?) fields + if field_path[0] in (self.env.context.get('name_create_enabled_fieds') or {}): + creatable_models.add(model_fields[field_path[0]].comodel_name) + for field_name in field_path: + if field_name in (None, 'id', '.id'): + break + + if isinstance(model_fields[field_name], odoo.fields.One2many): + comodel = model_fields[field_name].comodel_name + creatable_models.add(comodel) + model_fields = self.env[comodel]._fields + + def flush(*, xml_id=None, model=None): + if not batch: + return + + assert not (xml_id and model), \ + "flush can specify *either* an external id or a model, not both" + + if xml_id and xml_id not in batch_xml_ids: + if xml_id not in self.env: + return + if model and model not in creatable_models: + return + + data_list = [ + dict(xml_id=xid, values=vals, info=info, noupdate=noupdate) + for xid, vals, info in batch + ] + batch.clear() + batch_xml_ids.clear() + + # try to create in batch + try: + with cr.savepoint(): + recs = self._load_records(data_list, mode == 'update') + ids.extend(recs.ids) + return + except psycopg2.InternalError as e: + # broken transaction, exit and hope the source error was already logged + if not any(message['type'] == 'error' for message in messages): + info = data_list[0]['info'] + messages.append(dict(info, type='error', message=_(u"Unknown database error: '%s'", e))) + return + except Exception: + pass + + errors = 0 + # try again, this time record by record + for i, rec_data in enumerate(data_list, 1): + try: + with cr.savepoint(): + rec = self._load_records([rec_data], mode == 'update') + ids.append(rec.id) + except psycopg2.Warning as e: + info = rec_data['info'] + messages.append(dict(info, type='warning', message=str(e))) + except psycopg2.Error as e: + info = rec_data['info'] + messages.append(dict(info, type='error', **PGERROR_TO_OE[e.pgcode](self, fg, info, e))) + # Failed to write, log to messages, rollback savepoint (to + # avoid broken transaction) and keep going + errors += 1 + except Exception as e: + _logger.debug("Error while loading record", exc_info=True) + info = rec_data['info'] + message = (_(u'Unknown error during import:') + u' %s: %s' % (type(e), e)) + moreinfo = _('Resolve other errors first') + messages.append(dict(info, type='error', message=message, moreinfo=moreinfo)) + # Failed for some reason, perhaps due to invalid data supplied, + # rollback savepoint and keep going + errors += 1 + if errors >= 10 and (errors >= i / 10): + messages.append({ + 'type': 'warning', + 'message': _(u"Found more than 10 errors and more than one error per 10 records, interrupted to avoid showing too many errors.") + }) + break + + # make 'flush' available to the methods below, in the case where XMLID + # resolution fails, for instance + flush_self = self.with_context(import_flush=flush) + + # TODO: break load's API instead of smuggling via context? + limit = self._context.get('_import_limit') + if limit is None: + limit = float('inf') + extracted = flush_self._extract_records(fields, data, log=messages.append, limit=limit) + + converted = flush_self._convert_records(extracted, log=messages.append) + + info = {'rows': {'to': -1}} + for id, xid, record, info in converted: + if xid: + xid = xid if '.' in xid else "%s.%s" % (current_module, xid) + batch_xml_ids.add(xid) + # + # log csv records + openupgrade_log.log_xml_id(self.env.cr, current_module, xid) + # + elif id: + record['id'] = id + batch.append((xid, record, info)) + + flush() + if any(message['type'] == 'error' for message in messages): + cr.execute('ROLLBACK TO SAVEPOINT model_load') + ids = False + # cancel all changes done to the registry/ormcache + self.pool.reset_changes() + + nextrow = info['rows']['to'] + 1 + if nextrow < limit: + nextrow = 0 + return { + 'ids': ids, + 'messages': messages, + 'nextrow': nextrow, + } + +BaseModel.load = _load diff --git a/openupgrade_framework/odoo_patch/odoo/modules/__init__.py b/openupgrade_framework/odoo_patch/odoo/modules/__init__.py new file mode 100644 index 000000000000..90de5b4ff4e7 --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/modules/__init__.py @@ -0,0 +1,12 @@ +# Minor changes. (call to safe_eval changed) +# otherwise : adapted to V14 +from . import graph + +# A lot of changes in the core functions. +from . import loading + +# Adapted to V14 +from . import migration + +# Adapted to V14 +from . import registry diff --git a/openupgrade_framework/odoo_patch/odoo/modules/graph.py b/openupgrade_framework/odoo_patch/odoo/modules/graph.py new file mode 100644 index 000000000000..b0bedef3ea62 --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/modules/graph.py @@ -0,0 +1,108 @@ +# flake8: noqa +# pylint: skip-file + +import logging +import odoo +import odoo.tools as tools +from odoo.tools.safe_eval import safe_eval + +from odoo.modules.graph import Graph + +_logger = logging.getLogger(__name__) + + +if True: + + def _update_from_db(self, cr): + if not len(self): + return + # update the graph with values from the database (if exist) + ## First, we set the default values for each package in graph + additional_data = {key: {'id': 0, 'state': 'uninstalled', 'dbdemo': False, 'installed_version': None} for key in self.keys()} + ## Then we get the values from the database + cr.execute('SELECT name, id, state, demo AS dbdemo, latest_version AS installed_version' + ' FROM ir_module_module' + ' WHERE name IN %s',(tuple(additional_data),) + ) + + ## and we update the default values with values from the database + additional_data.update((x['name'], x) for x in cr.dictfetchall()) + + # + # Prevent reloading of demo data from the new version on major upgrade + if ('base' in self and additional_data['base']['dbdemo'] and + additional_data['base']['installed_version'] < + odoo.release.major_version): + cr.execute("UPDATE ir_module_module SET demo = false") + for data in additional_data.values(): + data['dbdemo'] = False + # + + for package in self.values(): + for k, v in additional_data[package.name].items(): + setattr(package, k, v) + + + def _add_modules(self, cr, module_list, force=None): + if force is None: + force = [] + packages = [] + len_graph = len(self) + + # + # force additional dependencies for the upgrade process if given + # in config file + forced_deps = tools.config.get_misc('openupgrade', 'force_deps', '{}') + forced_deps = tools.config.get_misc('openupgrade', + 'force_deps_' + odoo.release.version, + forced_deps) + forced_deps = safe_eval(forced_deps) + # + + for module in module_list: + # This will raise an exception if no/unreadable descriptor file. + # NOTE The call to load_information_from_description_file is already + # done by db.initialize, so it is possible to not do it again here. + info = odoo.modules.module.load_information_from_description_file(module) + if info and info['installable']: + # + info['depends'].extend(forced_deps.get(module, [])) + # + packages.append((module, info)) # TODO directly a dict, like in get_modules_with_version + elif module != 'studio_customization': + _logger.warning('module %s: not installable, skipped', module) + + dependencies = dict([(p, info['depends']) for p, info in packages]) + current, later = set([p for p, info in packages]), set() + + while packages and current > later: + package, info = packages[0] + deps = info['depends'] + + # if all dependencies of 'package' are already in the graph, add 'package' in the graph + if all(dep in self for dep in deps): + if not package in current: + packages.pop(0) + continue + later.clear() + current.remove(package) + node = self.add_node(package, info) + for kind in ('init', 'demo', 'update'): + if package in tools.config[kind] or 'all' in tools.config[kind] or kind in force: + setattr(node, kind, True) + else: + later.add(package) + packages.append((package, info)) + packages.pop(0) + + self.update_from_db(cr) + + for package in later: + unmet_deps = [p for p in dependencies[package] if p not in self] + _logger.info('module %s: Unmet dependencies: %s', package, ', '.join(unmet_deps)) + + return len(self) - len_graph + + +Graph.update_from_db = _update_from_db +Graph.add_modules = _add_modules diff --git a/openupgrade_framework/odoo_patch/odoo/modules/loading.py b/openupgrade_framework/odoo_patch/odoo/modules/loading.py new file mode 100644 index 000000000000..eb25c80ade5e --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/modules/loading.py @@ -0,0 +1,556 @@ +# flake8: noqa +# pylint: skip-file + +import itertools +import logging +import sys +import time + +import odoo +import odoo.tools as tools +from odoo import api, SUPERUSER_ID +from odoo.modules import loading +from odoo.modules.module import adapt_version, load_openerp_module, initialize_sys_path + +from odoo.modules.loading import load_data, load_demo, _check_module_names +from odoo.addons.openupgrade_framework.openupgrade import openupgrade_loading + +import os + +_logger = logging.getLogger(__name__) +_test_logger = logging.getLogger('odoo.tests') + + +def _load_module_graph(cr, graph, status=None, perform_checks=True, + skip_modules=None, report=None, models_to_check=None, upg_registry=None): + # + """Migrates+Updates or Installs all module nodes from ``graph`` + :param graph: graph of module nodes to load + :param status: deprecated parameter, unused, left to avoid changing signature in 8.0 + :param perform_checks: whether module descriptors should be checked for validity (prints warnings + for same cases) + :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped + :return: list of modules that were installed or updated + """ + if skip_modules is None: + skip_modules = [] + + if models_to_check is None: + models_to_check = set() + + processed_modules = [] + loaded_modules = [] + registry = odoo.registry(cr.dbname) + migrations = odoo.modules.migration.MigrationManager(cr, graph) + module_count = len(graph) + _logger.info('loading %d modules...', module_count) + + # + # suppress commits to have the upgrade of one module in just one transaction + cr.commit_org = cr.commit + cr.commit = lambda *args: None + cr.rollback_org = cr.rollback + cr.rollback = lambda *args: None + # + + # register, instantiate and initialize models for each modules + t0 = time.time() + loading_extra_query_count = odoo.sql_db.sql_counter + loading_cursor_query_count = cr.sql_log_count + + models_updated = set() + + for index, package in enumerate(graph, 1): + module_name = package.name + module_id = package.id + + # + if module_name in skip_modules or module_name in loaded_modules: + # + continue + + module_t0 = time.time() + module_cursor_query_count = cr.sql_log_count + module_extra_query_count = odoo.sql_db.sql_counter + + needs_update = ( + hasattr(package, "init") + or hasattr(package, "update") + or package.state in ("to install", "to upgrade") + ) + module_log_level = logging.DEBUG + if needs_update: + module_log_level = logging.INFO + _logger.log(module_log_level, 'Loading module %s (%d/%d)', module_name, index, module_count) + + if needs_update: + if package.name != 'base': + registry.setup_models(cr) + migrations.migrate_module(package, 'pre') + if package.name != 'base': + env = api.Environment(cr, SUPERUSER_ID, {}) + env['base'].flush() + + load_openerp_module(package.name) + + new_install = package.state == 'to install' + if new_install: + py_module = sys.modules['odoo.addons.%s' % (module_name,)] + pre_init = package.info.get('pre_init_hook') + if pre_init: + getattr(py_module, pre_init)(cr) + + model_names = registry.load(cr, package) + + mode = 'update' + if hasattr(package, 'init') or package.state == 'to install': + mode = 'init' + + loaded_modules.append(package.name) + if needs_update: + models_updated |= set(model_names) + models_to_check -= set(model_names) + registry.setup_models(cr) + # + # rebuild the local registry based on the loaded models + local_registry = {} + env = api.Environment(cr, SUPERUSER_ID, {}) + for model in env.values(): + if not model._auto: + continue + openupgrade_loading.log_model(model, local_registry) + openupgrade_loading.compare_registries( + cr, package.name, upg_registry, local_registry) + # + + registry.init_models(cr, model_names, {'module': package.name}, new_install) + elif package.state != 'to remove': + # The current module has simply been loaded. The models extended by this module + # and for which we updated the schema, must have their schema checked again. + # This is because the extension may have changed the model, + # e.g. adding required=True to an existing field, but the schema has not been + # updated by this module because it's not marked as 'to upgrade/to install'. + models_to_check |= set(model_names) & models_updated + + idref = {} + + if needs_update: + env = api.Environment(cr, SUPERUSER_ID, {}) + # Can't put this line out of the loop: ir.module.module will be + # registered by init_models() above. + module = env['ir.module.module'].browse(module_id) + + if perform_checks: + module._check() + + if package.state == 'to upgrade': + # upgrading the module information + module.write(module.get_values_from_terp(package.data)) + load_data(cr, idref, mode, kind='data', package=package) + demo_loaded = package.dbdemo = load_demo(cr, package, idref, mode) + cr.execute('update ir_module_module set demo=%s where id=%s', (demo_loaded, module_id)) + module.invalidate_cache(['demo']) + + # + # add 'try' block for logging exceptions + # as errors in post scripts seem to be dropped + try: + migrations.migrate_module(package, 'post') + except Exception as exc: + _logger.error('Error executing post migration script for module %s: %s', + package, exc) + raise + # + + # Update translations for all installed languages + overwrite = odoo.tools.config["overwrite_existing_translations"] + module.with_context(overwrite=overwrite)._update_translations() + + if package.name is not None: + registry._init_modules.add(package.name) + + if needs_update: + if new_install: + post_init = package.info.get('post_init_hook') + if post_init: + getattr(py_module, post_init)(cr, registry) + + if mode == 'update': + # validate the views that have not been checked yet + env['ir.ui.view']._validate_module_views(module_name) + + # need to commit any modification the module's installation or + # update made to the schema or data so the tests can run + # (separately in their own transaction) + # + # commit after processing every module as well, for + # easier debugging and continuing an interrupted migration + cr.commit_org() + # + # run tests + if os.environ.get('OPENUPGRADE_TESTS') and package.name is not None: + prefix = '.migrations' + registry.openupgrade_test_prefixes[package.name] = prefix + report.record_result(odoo.modules.module.run_unit_tests(module_name, openupgrade_prefix=prefix)) + # + # commit module_n state and version immediatly + # to avoid invalid database state if module_n+1 raises an + # exception + cr.commit_org() + # + + package.load_state = package.state + package.load_version = package.installed_version + package.state = 'installed' + for kind in ('init', 'demo', 'update'): + if hasattr(package, kind): + delattr(package, kind) + module.flush() + + extra_queries = odoo.sql_db.sql_counter - module_extra_query_count - test_queries + extras = [] + if test_queries: + extras.append(f'+{test_queries} test') + if extra_queries: + extras.append(f'+{extra_queries} other') + _logger.log( + module_log_level, "Module %s loaded in %.2fs%s, %s queries%s", + module_name, time.time() - module_t0, + f' (incl. {test_time:.2f}s test)' if test_time else '', + cr.sql_log_count - module_cursor_query_count, + f' ({", ".join(extras)})' if extras else '' + ) + if test_results and not test_results.wasSuccessful(): + _logger.error( + "Module %s: %d failures, %d errors of %d tests", + module_name, len(test_results.failures), len(test_results.errors), + test_results.testsRun + ) + + _logger.runbot("%s modules loaded in %.2fs, %s queries (+%s extra)", + len(graph), + time.time() - t0, + cr.sql_log_count - loading_cursor_query_count, + odoo.sql_db.sql_counter - loading_extra_query_count) # extra queries: testes, notify, any other closed cursor + + # + # restore commit method + cr.commit = cr.commit_org + cr.commit() + # + + return loaded_modules, processed_modules + + +def _load_marked_modules(cr, graph, states, force, progressdict, report, + loaded_modules, perform_checks, models_to_check=None, upg_registry=None): + # + """Loads modules marked with ``states``, adding them to ``graph`` and + ``loaded_modules`` and returns a list of installed/upgraded modules.""" + + if models_to_check is None: + models_to_check = set() + + processed_modules = [] + while True: + cr.execute("SELECT name from ir_module_module WHERE state IN %s" ,(tuple(states),)) + module_list = [name for (name,) in cr.fetchall() if name not in graph] + # + module_list = openupgrade_loading.add_module_dependencies(cr, module_list) + # + if not module_list: + break + graph.add_modules(cr, module_list, force) + _logger.debug('Updating graph with %d more modules', len(module_list)) + # + # add upg_registry + loaded, processed = _load_module_graph( + cr, graph, progressdict, report=report, skip_modules=loaded_modules, + perform_checks=perform_checks, models_to_check=models_to_check, + upg_registry=upg_registry, + ) + # + processed_modules.extend(processed) + loaded_modules.extend(loaded) + if not processed: + break + return processed_modules + + +def _load_modules(db, force_demo=False, status=None, update_module=False): + initialize_sys_path() + + force = [] + if force_demo: + force.append('demo') + + # + upg_registry = {} + # + + models_to_check = set() + + with db.cursor() as cr: + if not odoo.modules.db.is_initialized(cr): + if not update_module: + _logger.error("Database %s not initialized, you can force it with `-i base`", cr.dbname) + return + _logger.info("init db") + odoo.modules.db.initialize(cr) + update_module = True # process auto-installed modules + tools.config["init"]["all"] = 1 + if not tools.config['without_demo']: + tools.config["demo"]['all'] = 1 + + # This is a brand new registry, just created in + # odoo.modules.registry.Registry.new(). + registry = odoo.registry(cr.dbname) + + if 'base' in tools.config['update'] or 'all' in tools.config['update']: + cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed')) + + # STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps) + graph = odoo.modules.graph.Graph() + graph.add_module(cr, 'base', force) + if not graph: + _logger.critical('module base cannot be loaded! (hint: verify addons-path)') + raise ImportError('Module `base` cannot be loaded! (hint: verify addons-path)') + + # processed_modules: for cleanup step after install + # loaded_modules: to avoid double loading + report = registry._assertion_report + # + # add upg_registry + loaded_modules, processed_modules = _load_module_graph( + cr, graph, status, perform_checks=update_module, + report=report, models_to_check=models_to_check, upg_registry=upg_registry) + + # + load_lang = tools.config.pop('load_language') + if load_lang or update_module: + # some base models are used below, so make sure they are set up + registry.setup_models(cr) + + if load_lang: + for lang in load_lang.split(','): + tools.load_language(cr, lang) + + # STEP 2: Mark other modules to be loaded/updated + if update_module: + env = api.Environment(cr, SUPERUSER_ID, {}) + Module = env['ir.module.module'] + _logger.info('updating modules list') + Module.update_list() + + _check_module_names(cr, itertools.chain(tools.config['init'], tools.config['update'])) + + module_names = [k for k, v in tools.config['init'].items() if v] + if module_names: + modules = Module.search([('state', '=', 'uninstalled'), ('name', 'in', module_names)]) + if modules: + modules.button_install() + + module_names = [k for k, v in tools.config['update'].items() if v] + if module_names: + # + # in standard Odoo, '--update all' just means: + # '--update base + upward (installed) dependencies. This breaks + # the chain when new glue modules are encountered. + # E.g. purchase in 8.0 depends on stock_account and report, + # both of which are new. They may be installed, but purchase as + # an upward dependency is not selected for upgrade. + # Therefore, explicitely select all installed modules for + # upgrading in OpenUpgrade in that case. + domain = [('state', '=', 'installed')] + if 'all' not in module_names: + domain.append(('name', 'in', module_names)) + modules = Module.search(domain) + # + if modules: + modules.button_upgrade() + + cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base')) + Module.invalidate_cache(['state']) + Module.flush() + + # STEP 3: Load marked modules (skipping base which was done in STEP 1) + # IMPORTANT: this is done in two parts, first loading all installed or + # partially installed modules (i.e. installed/to upgrade), to + # offer a consistent system to the second part: installing + # newly selected modules. + # We include the modules 'to remove' in the first step, because + # they are part of the "currently installed" modules. They will + # be dropped in STEP 6 later, before restarting the loading + # process. + # IMPORTANT 2: We have to loop here until all relevant modules have been + # processed, because in some rare cases the dependencies have + # changed, and modules that depend on an uninstalled module + # will not be processed on the first pass. + # It's especially useful for migrations. + previously_processed = -1 + while previously_processed < len(processed_modules): + previously_processed = len(processed_modules) + # + # add upg_registry + processed_modules += _load_marked_modules(cr, graph, + ['installed', 'to upgrade', 'to remove'], + force, status, report, loaded_modules, update_module, models_to_check, upg_registry) + # + if update_module: + # + # add upg_registry + processed_modules += _load_marked_modules(cr, graph, + ['to install'], force, status, report, + loaded_modules, update_module, models_to_check, upg_registry) + # + # check that new module dependencies have been properly installed after a migration/upgrade + cr.execute("SELECT name from ir_module_module WHERE state IN ('to install', 'to upgrade')") + module_list = [name for (name,) in cr.fetchall()] + if module_list: + _logger.error("Some modules have inconsistent states, some dependencies may be missing: %s", sorted(module_list)) + + # check that all installed modules have been loaded by the registry after a migration/upgrade + cr.execute("SELECT name from ir_module_module WHERE state = 'installed' and name != 'studio_customization'") + module_list = [name for (name,) in cr.fetchall() if name not in graph] + if module_list: + _logger.error("Some modules are not loaded, some dependencies or manifest may be missing: %s", sorted(module_list)) + + registry.loaded = True + registry.setup_models(cr) + + # STEP 3.5: execute migration end-scripts + migrations = odoo.modules.migration.MigrationManager(cr, graph) + for package in graph: + migrations.migrate_module(package, 'end') + + # STEP 3.6: apply remaining constraints in case of an upgrade + registry.finalize_constraints() + + # STEP 4: Finish and cleanup installations + if processed_modules: + env = api.Environment(cr, SUPERUSER_ID, {}) + cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""") + for (model, name) in cr.fetchall(): + if model in registry and not registry[model]._abstract: + _logger.warning('The model %s has no access rules, consider adding one. E.g. access_%s,access_%s,model_%s,base.group_user,1,0,0,0', + model, model.replace('.', '_'), model.replace('.', '_'), model.replace('.', '_')) + + cr.execute("SELECT model from ir_model") + for (model,) in cr.fetchall(): + if model in registry: + env[model]._check_removed_columns(log=True) + elif _logger.isEnabledFor(logging.INFO): # more an info that a warning... + _logger.runbot("Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)", model) + + # Cleanup orphan records + env['ir.model.data']._process_end(processed_modules) + env['base'].flush() + + for kind in ('init', 'demo', 'update'): + tools.config[kind] = {} + + # STEP 5: Uninstall modules to remove + if update_module: + # Remove records referenced from ir_model_data for modules to be + # removed (and removed the references from ir_model_data). + cr.execute("SELECT name, id FROM ir_module_module WHERE state=%s", ('to remove',)) + modules_to_remove = dict(cr.fetchall()) + if modules_to_remove: + env = api.Environment(cr, SUPERUSER_ID, {}) + pkgs = reversed([p for p in graph if p.name in modules_to_remove]) + for pkg in pkgs: + uninstall_hook = pkg.info.get('uninstall_hook') + if uninstall_hook: + py_module = sys.modules['odoo.addons.%s' % (pkg.name,)] + getattr(py_module, uninstall_hook)(cr, registry) + + Module = env['ir.module.module'] + Module.browse(modules_to_remove.values()).module_uninstall() + # Recursive reload, should only happen once, because there should be no + # modules to remove next time + cr.commit() + _logger.info('Reloading registry once more after uninstalling modules') + api.Environment.reset() + registry = odoo.modules.registry.Registry.new( + cr.dbname, force_demo, status, update_module + ) + registry.check_tables_exist(cr) + cr.commit() + return registry + + # STEP 5.5: Verify extended fields on every model + # This will fix the schema of all models in a situation such as: + # - module A is loaded and defines model M; + # - module B is installed/upgraded and extends model M; + # - module C is loaded and extends model M; + # - module B and C depend on A but not on each other; + # The changes introduced by module C are not taken into account by the upgrade of B. + if models_to_check: + registry.init_models(cr, list(models_to_check), {'models_to_check': True}) + + # STEP 6: verify custom views on every model + if update_module: + env = api.Environment(cr, SUPERUSER_ID, {}) + env['res.groups']._update_user_groups_view() + View = env['ir.ui.view'] + for model in registry: + try: + View._validate_custom_views(model) + except Exception as e: + _logger.warning('invalid custom view(s) for model %s: %s', model, tools.ustr(e)) + + if report.wasSuccessful(): + _logger.info('Modules loaded.') + else: + _logger.error('At least one test failed when loading the modules.') + + # STEP 8: call _register_hook on every model + # This is done *exactly once* when the registry is being loaded. See the + # management of those hooks in `Registry.setup_models`: all the calls to + # setup_models() done here do not mess up with hooks, as registry.ready + # is False. + env = api.Environment(cr, SUPERUSER_ID, {}) + for model in env.values(): + model._register_hook() + env['base'].flush() + + # STEP 9: save installed/updated modules for post-install tests + registry.updated_modules += processed_modules + +loading.load_module_graph = _load_module_graph +loading.load_marked_modules = _load_marked_modules +loading.load_modules = _load_modules +odoo.modules.load_modules = _load_modules diff --git a/openupgrade_framework/odoo_patch/odoo/modules/migration.py b/openupgrade_framework/odoo_patch/odoo/modules/migration.py new file mode 100644 index 000000000000..0346c2b8c559 --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/modules/migration.py @@ -0,0 +1,118 @@ +# flake8: noqa +# pylint: skip-file + +import logging +import os +from os.path import join as opj +import odoo.release as release +from odoo.tools.parse_version import parse_version + +import odoo +from odoo.modules.migration import load_script +from odoo.modules import migration + +_logger = logging.getLogger(__name__) + + +if True: + def _migrate_module(self, pkg, stage): + assert stage in ('pre', 'post', 'end') + stageformat = { + 'pre': '[>%s]', + 'post': '[%s>]', + 'end': '[$%s]', + } + state = pkg.state if stage in ('pre', 'post') else getattr(pkg, 'load_state', None) + + # + # In openupgrade, also run migration scripts upon installation. + # We want to always pass in pre and post migration files and use a new + # argument in the migrate decorator (explained in the docstring) + # to decide if we want to do something if a new module is installed + # during the migration. + if not (hasattr(pkg, 'update') or state in ('to upgrade', 'to install')): + # + return + + def convert_version(version): + if version.count('.') >= 2: + return version # the version number already containt the server version + return "%s.%s" % (release.major_version, version) + + def _get_migration_versions(pkg, stage): + versions = sorted({ + ver + for lv in self.migrations[pkg.name].values() + for ver, lf in lv.items() + if lf + }, key=lambda k: parse_version(convert_version(k))) + if "0.0.0" in versions: + # reorder versions + versions.remove("0.0.0") + if stage == "pre": + versions.insert(0, "0.0.0") + else: + versions.append("0.0.0") + return versions + + def _get_migration_files(pkg, version, stage): + """ return a list of migration script files + """ + m = self.migrations[pkg.name] + lst = [] + + mapping = { + 'module': opj(pkg.name, 'migrations'), + 'module_upgrades': opj(pkg.name, 'upgrades'), + } + + for path in odoo.upgrade.__path__: + if os.path.exists(opj(path, pkg.name)): + mapping['upgrade'] = opj(path, pkg.name) + break + + for x in mapping: + if version in m.get(x): + for f in m[x][version]: + if not f.startswith(stage + '-'): + continue + lst.append(opj(mapping[x], version, f)) + lst.sort() + return lst + + installed_version = getattr(pkg, 'load_version', pkg.installed_version) or '' + parsed_installed_version = parse_version(installed_version) + current_version = parse_version(convert_version(pkg.data['version'])) + + versions = _get_migration_versions(pkg, stage) + + for version in versions: + if ((version == "0.0.0" and parsed_installed_version < current_version) + or parsed_installed_version < parse_version(convert_version(version)) <= current_version): + + strfmt = {'addon': pkg.name, + 'stage': stage, + 'version': stageformat[stage] % version, + } + + for pyfile in _get_migration_files(pkg, version, stage): + name, ext = os.path.splitext(os.path.basename(pyfile)) + if ext.lower() != '.py': + continue + mod = None + try: + mod = load_script(pyfile, name) + _logger.info('module %(addon)s: Running migration %(version)s %(name)s' % dict(strfmt, name=mod.__name__)) + migrate = mod.migrate + except ImportError: + _logger.exception('module %(addon)s: Unable to load %(stage)s-migration file %(file)s' % dict(strfmt, file=pyfile)) + raise + except AttributeError: + _logger.error('module %(addon)s: Each %(stage)s-migration file must have a "migrate(cr, installed_version)" function' % strfmt) + else: + migrate(self.cr, installed_version) + finally: + if mod: + del mod + +migration.migrate_module = _migrate_module diff --git a/openupgrade_framework/odoo_patch/odoo/modules/registry.py b/openupgrade_framework/odoo_patch/odoo/modules/registry.py new file mode 100644 index 000000000000..4c5f50d4e714 --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/modules/registry.py @@ -0,0 +1,58 @@ +# flake8: noqa +# pylint: skip-file + +from collections import deque +from contextlib import closing +import odoo +from odoo.tools.lru import LRU + +from odoo.modules import registry + + +if True: + + def _init(self, db_name): + self.models = {} # model name/model instance mapping + self._sql_constraints = set() + self._init = True + self._assertion_report = odoo.tests.runner.OdooTestResult() + self._fields_by_model = None + self._ordinary_tables = None + self._constraint_queue = deque() + self.__cache = LRU(8192) + + # modules fully loaded (maintained during init phase by `loading` module) + self._init_modules = set() + self.updated_modules = [] # installed/updated modules + # + self.openupgrade_test_prefixes = {} + # + self.loaded_xmlids = set() + + self.db_name = db_name + self._db = odoo.sql_db.db_connect(db_name) + + # cursor for test mode; None means "normal" mode + self.test_cr = None + self.test_lock = None + + # Indicates that the registry is + self.loaded = False # whether all modules are loaded + self.ready = False # whether everything is set up + + # Inter-process signaling: + # The `base_registry_signaling` sequence indicates the whole registry + # must be reloaded. + # The `base_cache_signaling sequence` indicates all caches must be + # invalidated (i.e. cleared). + self.registry_sequence = None + self.cache_sequence = None + + # Flags indicating invalidation of the registry or the cache. + self.registry_invalidated = False + self.cache_invalidated = False + + with closing(self.cursor()) as cr: + self.has_unaccent = odoo.modules.db.has_unaccent(cr) + +registry.init = _init diff --git a/openupgrade_framework/odoo_patch/odoo/service/__init__.py b/openupgrade_framework/odoo_patch/odoo/service/__init__.py new file mode 100644 index 000000000000..a96314d0f684 --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/service/__init__.py @@ -0,0 +1,4 @@ +# Import disabled, because the function run_unit_tests() +# disappeared in V14. +# TODO: OpenUpgrade Core maintainers : FIXME. +# from . import server diff --git a/openupgrade_framework/odoo_patch/odoo/service/server.py b/openupgrade_framework/odoo_patch/odoo/service/server.py new file mode 100644 index 000000000000..a2a998e69df3 --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/service/server.py @@ -0,0 +1,71 @@ +# flake8: noqa +# pylint: skip-file + +import logging +import os +import time + +import odoo +from odoo.tools import config +from odoo.modules.registry import Registry + +from odoo.service import server +from odoo.service.server import load_test_file_py + +_logger = logging.getLogger(__name__) + + +def preload_registries(dbnames): + """ Preload a registries, possibly run a test file.""" + # TODO: move all config checks to args dont check tools.config here + dbnames = dbnames or [] + rc = 0 + for dbname in dbnames: + try: + update_module = config['init'] or config['update'] + registry = Registry.new(dbname, update_module=update_module) + + # run test_file if provided + if config['test_file']: + test_file = config['test_file'] + if not os.path.isfile(test_file): + _logger.warning('test file %s cannot be found', test_file) + elif not test_file.endswith('py'): + _logger.warning('test file %s is not a python file', test_file) + else: + _logger.info('loading test file %s', test_file) + with odoo.api.Environment.manage(): + load_test_file_py(registry, test_file) + + # run post-install tests + if config['test_enable']: + t0 = time.time() + t0_sql = odoo.sql_db.sql_counter + module_names = (registry.updated_modules if update_module else + sorted(registry._init_modules)) + _logger.info("Starting post tests") + tests_before = registry._assertion_report.testsRun + with odoo.api.Environment.manage(): + for module_name in module_names: + result = loader.run_suite(loader.make_suite(module_name, 'post_install'), module_name) + registry._assertion_report.update(result) + # + # run deferred unit tests + for module_name, prefix in registry.openupgrade_test_prefixes: + result = run_unit_tests(module_name, position='post_install', openupgrade_prefix=prefix) + registry._assertion_report.record_result(result) + # + _logger.info("%d post-tests in %.2fs, %s queries", + registry._assertion_report.testsRun - tests_before, + time.time() - t0, + odoo.sql_db.sql_counter - t0_sql) + + if not registry._assertion_report.wasSuccessful(): + rc += 1 + except Exception: + _logger.critical('Failed to initialize database `%s`.', dbname, exc_info=True) + return -1 + return rc + + +server.preload_registries = preload_registries diff --git a/openupgrade_framework/odoo_patch/odoo/tools/__init__.py b/openupgrade_framework/odoo_patch/odoo/tools/__init__.py new file mode 100644 index 000000000000..6ad156515dc3 --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/tools/__init__.py @@ -0,0 +1,2 @@ +from . import convert +from . import view_validation diff --git a/openupgrade_framework/odoo_patch/odoo/tools/convert.py b/openupgrade_framework/odoo_patch/odoo/tools/convert.py new file mode 100644 index 000000000000..49531bfc429e --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/tools/convert.py @@ -0,0 +1,23 @@ +# flake8: noqa +# pylint: skip-file + +from odoo.addons.openupgrade_framework.openupgrade import openupgrade_log + +from odoo.tools.convert import xml_import + +if True: + + def __test_xml_id(self, xml_id): + if '.' in xml_id: + module, id = xml_id.split('.', 1) + assert '.' not in id, """The ID reference "%s" must contain +maximum one dot. They are used to refer to other modules ID, in the +form: module.record_id""" % (xml_id,) + if module != self.module: + modcnt = self.env['ir.module.module'].search_count([('name', '=', module), ('state', '=', 'installed')]) + assert modcnt == 1, """The ID "%s" refers to an uninstalled module""" % (xml_id,) + + # OpenUpgrade: log entry of XML imports + openupgrade_log.log_xml_id(self.env.cr, self.module, xml_id) + +xml_import._test_xml_id = __test_xml_id diff --git a/openupgrade_framework/odoo_patch/odoo/tools/view_validation.py b/openupgrade_framework/odoo_patch/odoo/tools/view_validation.py new file mode 100644 index 000000000000..e6c8243241af --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/tools/view_validation.py @@ -0,0 +1,29 @@ +# flake8: noqa +# pylint: skip-file + +# from odoo.addons.openupgrade_framework.openupgrade import openupgrade_log + +from odoo.tools import view_validation +from odoo.tools.view_validation import _validators, _logger + + +def _valid_view(arch, **kwargs): + for pred in _validators[arch.tag]: + # + # Do not raise blocking error, because it's normal to + # have inconsistent views in an openupgrade process + check = pred(arch, **kwargs) or 'Warning' + # + if not check: + _logger.error("Invalid XML: %s", pred.__doc__) + return False + if check == "Warning": + # + # Don't show this warning as useless and too much verbose + # _logger.warning("Invalid XML: %s", pred.__doc__) + # + return "Warning" + return True + + +view_validation.valid_view = _valid_view diff --git a/openupgrade_framework/openupgrade/__init__.py b/openupgrade_framework/openupgrade/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openupgrade_framework/openupgrade/openupgrade_loading.py b/openupgrade_framework/openupgrade/openupgrade_loading.py new file mode 100644 index 000000000000..ca3e1d43067d --- /dev/null +++ b/openupgrade_framework/openupgrade/openupgrade_loading.py @@ -0,0 +1,318 @@ +# Copyright 2011-2015 Therp BV +# Copyright 2016-2019 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +# flake8: noqa: C901 + +import logging + +from openupgradelib.openupgrade_tools import table_exists + +from odoo import release +from odoo.modules.module import get_module_path +from odoo.tools.safe_eval import safe_eval +from odoo.tools.config import config + +# A collection of functions used in +# odoo/modules/loading.py + +logger = logging.getLogger("OpenUpgrade") + + +def add_module_dependencies(cr, module_list): + """ + Select (new) dependencies from the modules in the list + so that we can inject them into the graph at upgrade + time. Used in the modified OpenUpgrade Server, + not to be called from migration scripts + + Also take the OpenUpgrade configuration directives 'forced_deps' + and 'autoinstall' into account. From any additional modules + that these directives can add, the dependencies are added as + well (but these directives are not checked for the occurrence + of any of the dependencies). + """ + if not module_list: + return module_list + + modules_in = list(module_list) + forced_deps = safe_eval( + config.get_misc( + "openupgrade", + "forced_deps_" + release.version, + config.get_misc("openupgrade", "forced_deps", "{}"), + ) + ) + + autoinstall = safe_eval( + config.get_misc( + "openupgrade", + "autoinstall_" + release.version, + config.get_misc("openupgrade", "autoinstall", "{}"), + ) + ) + + for module in list(module_list): + module_list += forced_deps.get(module, []) + module_list += autoinstall.get(module, []) + + module_list = list(set(module_list)) + + dependencies = module_list + while dependencies: + cr.execute( + """ + SELECT DISTINCT dep.name + FROM + ir_module_module, + ir_module_module_dependency dep + WHERE + module_id = ir_module_module.id + AND ir_module_module.name in %s + AND dep.name not in %s + """, + ( + tuple(dependencies), + tuple(module_list), + ), + ) + + dependencies = [x[0] for x in cr.fetchall()] + module_list += dependencies + + # Select auto_install modules of which all dependencies + # are fulfilled based on the modules we know are to be + # installed + cr.execute( + """ + SELECT name from ir_module_module WHERE state IN %s + """, + (("installed", "to install", "to upgrade"),), + ) + modules = list(set(module_list + [row[0] for row in cr.fetchall()])) + cr.execute( + """ + SELECT name from ir_module_module m + WHERE auto_install IS TRUE + AND state = 'uninstalled' + AND NOT EXISTS( + SELECT id FROM ir_module_module_dependency d + WHERE d.module_id = m.id + AND name NOT IN %s) + """, + (tuple(modules),), + ) + auto_modules = [row[0] for row in cr.fetchall() if get_module_path(row[0])] + if auto_modules: + logger.info("Selecting autoinstallable modules %s", ",".join(auto_modules)) + module_list += auto_modules + + # Set proper state for new dependencies so that any init scripts are run + cr.execute( + """ + UPDATE ir_module_module SET state = 'to install' + WHERE name IN %s AND name NOT IN %s AND state = 'uninstalled' + """, + (tuple(module_list), tuple(modules_in)), + ) + return module_list + + +def log_model(model, local_registry): + """ + OpenUpgrade: Store the characteristics of the BaseModel and its fields + in the local registry, so that we can compare changes with the + main registry + """ + + if not model._name: + return + + typemap = {"monetary": "float"} + + # Deferred import to prevent import loop + from odoo import models + + # persistent models only + if isinstance(model, models.TransientModel): + return + + def isfunction(model, k): + if ( + model._fields[k].compute + and not model._fields[k].related + and not model._fields[k].company_dependent + ): + return "function" + return "" + + def isproperty(model, k): + if model._fields[k].company_dependent: + return "property" + return "" + + def isrelated(model, k): + if model._fields[k].related: + return "related" + return "" + + def _get_relation(v): + if v.type in ("many2many", "many2one", "one2many"): + return v.comodel_name + elif v.type == "many2one_reference": + return v.model_field + else: + return "" + + model_registry = local_registry.setdefault(model._name, {}) + if model._inherits: + model_registry["_inherits"] = {"_inherits": str(model._inherits)} + for k, v in model._fields.items(): + properties = { + "type": typemap.get(v.type, v.type), + "isfunction": isfunction(model, k), + "isproperty": isproperty(model, k), + "isrelated": isrelated(model, k), + "relation": _get_relation(v), + "table": v.relation if v.type == "many2many" else "", + "required": v.required and "required" or "", + "stored": v.store and "stored" or "", + "selection_keys": "", + "req_default": "", + "hasdefault": model._fields[k].default and "hasdefault" or "", + "inherits": "", + } + if v.type == "selection": + if isinstance(v.selection, (tuple, list)): + properties["selection_keys"] = str(sorted([x[0] for x in v.selection])) + else: + properties["selection_keys"] = "function" + elif v.type == "binary": + properties["attachment"] = str(getattr(v, "attachment", False)) + default = model._fields[k].default + if v.required and default: + if ( + callable(default) + or isinstance(default, str) + and getattr(model._fields[k], default, False) + and callable(getattr(model._fields[k], default)) + ): + # todo: in OpenERP 5 (and in 6 as well), + # literals are wrapped in a lambda function + properties["req_default"] = "function" + else: + properties["req_default"] = str(default) + for key, value in properties.items(): + if value: + model_registry.setdefault(k, {})[key] = value + + +def get_record_id(cr, module, model, field, mode): + """ + OpenUpgrade: get or create the id from the record table matching + the key parameter values + """ + cr.execute( + "SELECT id FROM openupgrade_record " + "WHERE module = %s AND model = %s AND " + "field = %s AND mode = %s AND type = %s", + (module, model, field, mode, "field"), + ) + record = cr.fetchone() + if record: + return record[0] + cr.execute( + "INSERT INTO openupgrade_record " + "(module, model, field, mode, type) " + "VALUES (%s, %s, %s, %s, %s)", + (module, model, field, mode, "field"), + ) + cr.execute( + "SELECT id FROM openupgrade_record " + "WHERE module = %s AND model = %s AND " + "field = %s AND mode = %s AND type = %s", + (module, model, field, mode, "field"), + ) + return cr.fetchone()[0] + + +def compare_registries(cr, module, registry, local_registry): + """ + OpenUpgrade: Compare the local registry with the global registry, + log any differences and merge the local registry with + the global one. + """ + if not table_exists(cr, "openupgrade_record"): + return + for model, flds in local_registry.items(): + registry.setdefault(model, {}) + for field, attributes in flds.items(): + old_field = registry[model].setdefault(field, {}) + mode = old_field and "modify" or "create" + record_id = False + for key, value in attributes.items(): + if key not in old_field or old_field[key] != value: + if not record_id: + record_id = get_record_id(cr, module, model, field, mode) + cr.execute( + "SELECT id FROM openupgrade_attribute " + "WHERE name = %s AND value = %s AND " + "record_id = %s", + (key, value, record_id), + ) + if not cr.fetchone(): + cr.execute( + "INSERT INTO openupgrade_attribute " + "(name, value, record_id) VALUES (%s, %s, %s)", + (key, value, record_id), + ) + old_field[key] = value + + +def update_field_xmlid(model, field): + """OpenUpgrade edit start: In rare cases, an old module defined a field + on a model that is not defined in another module earlier in the + chain of inheritance. Then we need to assign the ir.model.fields' + xmlid to this other module, otherwise the column would be dropped + when uninstalling the first module. + An example is res.partner#display_name defined in 7.0 by + account_report_company, but now the field belongs to the base + module + Given that we arrive here in order of inheritance, we simply check + if the field's xmlid belongs to a module already loaded, and if not, + update the record with the correct module name.""" + model.env.cr.execute( + "SELECT f.*, d.module, d.id as xmlid_id, d.name as xmlid " + "FROM ir_model_fields f LEFT JOIN ir_model_data d " + "ON f.id=d.res_id and d.model='ir.model.fields' WHERE f.model=%s", + (model._name,), + ) + for rec in model.env.cr.dictfetchall(): + if ( + "module" in model.env.context + and rec["module"] + and rec["name"] in model._fields.keys() + and rec["module"] != model.env.context["module"] + and rec["module"] not in model.env.registry._init_modules + ): + logging.getLogger(__name__).info( + "Moving XMLID for ir.model.fields record of %s#%s " "from %s to %s", + model._name, + rec["name"], + rec["module"], + model.env.context["module"], + ) + model.env.cr.execute( + "SELECT id FROM ir_model_data WHERE module=%(module)s " + "AND name=%(xmlid)s", + dict(rec, module=model.env.context["module"]), + ) + if model.env.cr.fetchone(): + logging.getLogger(__name__).info( + "Aborting, an XMLID for this module already exists." + ) + continue + model.env.cr.execute( + "UPDATE ir_model_data SET module=%(module)s " "WHERE id=%(xmlid_id)s", + dict(rec, module=model.env.context["module"]), + ) diff --git a/openupgrade_framework/openupgrade/openupgrade_log.py b/openupgrade_framework/openupgrade/openupgrade_log.py new file mode 100644 index 000000000000..81c8916738fb --- /dev/null +++ b/openupgrade_framework/openupgrade/openupgrade_log.py @@ -0,0 +1,60 @@ +# coding: utf-8 +# Copyright 2011-2015 Therp BV +# Copyright 2016 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from openupgradelib.openupgrade_tools import table_exists + + +def log_xml_id(cr, module, xml_id): + """ + Log xml_ids at load time in the records table. + Called from tools/convert.py:xml_import._test_xml_id() + + # Catcha's + - The module needs to be loaded with 'init', or the calling method + won't be called. This can be brought about by installing the + module or updating the 'state' field of the module to 'to install' + or call the server with '--init ' and the database argument. + + - Do you get the right results immediately when installing the module? + No, sorry. This method retrieves the model from the ir_model_table, but + when the xml id is encountered for the first time, this method is called + before the item is present in this table. Therefore, you will not + get any meaningful results until the *second* time that you 'init' + the module. + + - The good news is that the openupgrade_records module that comes + with this distribution allows you to deal with all of this with + one click on the menu item Settings -> Customizations -> + Database Structure -> OpenUpgrade -> Generate Records + + - You cannot reinitialize the modules in your production database + and expect to keep working on it happily ever after. Do not perform + this routine on your production database. + + :param module: The module that contains the xml_id + :param xml_id: the xml_id, with or without 'module.' prefix + """ + if not table_exists(cr, 'openupgrade_record'): + return + if '.' not in xml_id: + xml_id = '%s.%s' % (module, xml_id) + cr.execute( + "SELECT model FROM ir_model_data " + "WHERE module = %s AND name = %s", + xml_id.split('.')) + record = cr.fetchone() + if not record: + print("Cannot find xml_id %s" % xml_id) + return + else: + cr.execute( + "SELECT id FROM openupgrade_record " + "WHERE module=%s AND model=%s AND name=%s AND type=%s", + (module, record[0], xml_id, 'xmlid')) + if not cr.fetchone(): + cr.execute( + "INSERT INTO openupgrade_record " + "(module, model, name, type) values(%s, %s, %s, %s)", + (module, record[0], xml_id, 'xmlid')) diff --git a/openupgrade_framework/readme/CONFIGURE.rst b/openupgrade_framework/readme/CONFIGURE.rst new file mode 100644 index 000000000000..bb245fb3b72a --- /dev/null +++ b/openupgrade_framework/readme/CONFIGURE.rst @@ -0,0 +1,7 @@ +To use this module, do not install it. Instead, you should add the name in your +``odoo.cfg`` module : + +.. code-block:: shell + + [options] + server_wide_modules = web,openupgrade_framework diff --git a/openupgrade_framework/readme/CONTRIBUTORS.rst b/openupgrade_framework/readme/CONTRIBUTORS.rst new file mode 100644 index 000000000000..d693b699fbb2 --- /dev/null +++ b/openupgrade_framework/readme/CONTRIBUTORS.rst @@ -0,0 +1,2 @@ +* Stefan Rijnhart +* Sylvain LE GAL diff --git a/openupgrade_framework/readme/DESCRIPTION.rst b/openupgrade_framework/readme/DESCRIPTION.rst new file mode 100644 index 000000000000..efceae7e621b --- /dev/null +++ b/openupgrade_framework/readme/DESCRIPTION.rst @@ -0,0 +1,2 @@ +This module is a technical module, to allow to make migrations between +major versions of Odoo. diff --git a/openupgrade_framework/readme/DEVELOP.rst b/openupgrade_framework/readme/DEVELOP.rst new file mode 100644 index 000000000000..44c17e65d41d --- /dev/null +++ b/openupgrade_framework/readme/DEVELOP.rst @@ -0,0 +1,60 @@ +This module contains two folders: + + +odoo_patch +---------- + +This folder contains python files, that correspond to python files present +in the folder ``odoo`` of the Odoo project. + +it contains a lot of monkey patches, to make working an upgrade +between two major versions. +To see the patches added, you can use ``meld`` tools: + +``meld PATH_TO_ODOO_FOLDER/odoo/ PATH_TO_OPENUPGRADE_FRAMEWORK_MODULE/odoo_patch`` + + +To make more easy the diff analysis : + +* Make sure the python files has the same path as the original one. + +* Keep the same indentation as the original file. (using ``if True:`` if required) + +* Add the following two lines at the beginning of your file, to avoid flake8 / pylint + errors + +.. code-block:: python + + # flake8: noqa + # pylint: skip-file + +* When you want to change the code. add the following tags: + + * For an addition: + +.. code-block:: python + + # + some code... + # + + * For a change: + +.. code-block:: python + + # + some code... + # + + * For a removal: + +.. code-block:: python + + # + # Comment the code, instead of removing it. + # + +openupgrade +----------- + +Contains extra functions, called by the patches introduced in the first folder. diff --git a/openupgrade_records/README.rst b/openupgrade_records/README.rst new file mode 100644 index 000000000000..ac9f49126565 --- /dev/null +++ b/openupgrade_records/README.rst @@ -0,0 +1,8 @@ +=============================== +OpenUpgrade Database Comparison +=============================== + +.. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! diff --git a/openupgrade_records/__init__.py b/openupgrade_records/__init__.py new file mode 100644 index 000000000000..e28ed8d31343 --- /dev/null +++ b/openupgrade_records/__init__.py @@ -0,0 +1,3 @@ +from . import models +from . import wizards +from . import blacklist diff --git a/openupgrade_records/__manifest__.py b/openupgrade_records/__manifest__.py new file mode 100644 index 000000000000..4de38b2bdefe --- /dev/null +++ b/openupgrade_records/__manifest__.py @@ -0,0 +1,26 @@ +# Copyright 2011-2015 Therp BV +# Copyright 2016 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +{ + "name": "OpenUpgrade Database Comparison", + "summary": """Generate the database analysis files that indicate how the + Odoo data model and module data have changed between two versions of Odoo.""", + "version": "14.0.1.0.0", + "category": "Migration", + "author": "Odoo Community Association (OCA), Therp BV, Opener B.V., GRAP", + "website": "https://github.com/OCA/openupgrade-framework", + "data": [ + "views/menu.xml", + "views/openupgrade_comparison_config.xml", + "views/openupgrade_record.xml", + "wizards/openupgrade_analysis_wizard.xml", + "wizards/openupgrade_generate_records_wizard.xml", + "wizards/openupgrade_install_all_wizard.xml", + "security/ir.model.access.csv", + ], + "installable": True, + "external_dependencies": { + "python": ["odoorpc", "openupgradelib"], + }, + "license": "AGPL-3", +} diff --git a/openupgrade_records/apriori.py b/openupgrade_records/apriori.py new file mode 100644 index 000000000000..0144fc80a1b1 --- /dev/null +++ b/openupgrade_records/apriori.py @@ -0,0 +1,25 @@ +""" Encode any known changes to the database here +to help the matching process +""" + +renamed_modules = { + # Odoo + # OCA/... +} + +merged_modules = { + # Odoo + # OCA/... +} + +# only used here for openupgrade_records analysis: +renamed_models = { + # Odoo + # OCA/... +} + +# only used here for openupgrade_records analysis: +merged_models = { + # Odoo + # OCA/... +} diff --git a/openupgrade_records/blacklist.py b/openupgrade_records/blacklist.py new file mode 100644 index 000000000000..1b7972aebbc5 --- /dev/null +++ b/openupgrade_records/blacklist.py @@ -0,0 +1,12 @@ +BLACKLIST_MODULES = [ + # the hw_* modules are not affected by a migration as they don't + # contain any ORM functionality, but they do start up threads that + # delay the process and spit out annoying log messages continously. + "hw_drivers", + "hw_escpos", + "hw_posbox_homepage", + # Modules that seems bugged + "l10n_bo", + # Doesn't have sense to analyse this module that contains patches + "openupgrade_framework", +] diff --git a/openupgrade_records/compare.py b/openupgrade_records/compare.py new file mode 100644 index 000000000000..8cad747a0bb4 --- /dev/null +++ b/openupgrade_records/compare.py @@ -0,0 +1,483 @@ +# Copyright 2011-2015 Therp BV +# Copyright 2015-2016 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# flake8: noqa: C901 +##################################################################### +# library providing a function to analyse two progressive database +# layouts from the OpenUpgrade server. +##################################################################### + +import collections +import copy + +from . import apriori + + +def module_map(module): + return apriori.renamed_modules.get( + module, apriori.merged_modules.get(module, module) + ) + + +def model_rename_map(model): + return apriori.renamed_models.get(model, model) + + +def model_map(model): + return apriori.renamed_models.get(model, apriori.merged_models.get(model, model)) + + +def inv_model_map(model): + inv_model_map_dict = {v: k for k, v in apriori.renamed_models.items()} + return inv_model_map_dict.get(model, model) + + +IGNORE_FIELDS = [ + "create_date", + "create_uid", + "id", + "write_date", + "write_uid", +] + + +def compare_records(dict_old, dict_new, fields): + """ + Check equivalence of two OpenUpgrade field representations + with respect to the keys in the 'fields' arguments. + Take apriori knowledge into account for mapped modules or + model names. + Return True of False. + """ + for field in fields: + if field == "module": + if module_map(dict_old["module"]) != dict_new["module"]: + return False + elif field == "model": + if model_rename_map(dict_old["model"]) != dict_new["model"]: + return False + elif field == "other_prefix": + if ( + dict_old["module"] != dict_old["prefix"] + or dict_new["module"] != dict_new["prefix"] + ): + return False + if dict_old["model"] == "ir.ui.view": + # basically, to avoid the assets_backend case + return False + elif dict_old[field] != dict_new[field]: + return False + return True + + +def search(item, item_list, fields): + """ + Find a match of a dictionary in a list of similar dictionaries + with respect to the keys in the 'fields' arguments. + Return the item if found or None. + """ + for other in item_list: + if not compare_records(item, other, fields): + continue + return other + # search for renamed fields + if "field" in fields: + for other in item_list: + if not item["field"] or item["field"] is not None or item["isproperty"]: + continue + if compare_records(dict(item, field=other["field"]), other, fields): + return other + return None + + +def fieldprint(old, new, field, text, reprs): + fieldrepr = "{} ({})".format(old["field"], old["type"]) + fullrepr = "{:<12} / {:<24} / {:<30}".format(old["module"], old["model"], fieldrepr) + if not text: + text = "{} is now '{}' ('{}')".format(field, new[field], old[field]) + if field == "relation": + text += " [nothing to do]" + reprs[module_map(old["module"])].append("{}: {}".format(fullrepr, text)) + if field == "module": + text = "previously in module %s" % old[field] + fullrepr = "{:<12} / {:<24} / {:<30}".format( + new["module"], old["model"], fieldrepr + ) + reprs[module_map(new["module"])].append("{}: {}".format(fullrepr, text)) + + +def report_generic(new, old, attrs, reprs): + for attr in attrs: + if attr == "required": + if old[attr] != new["required"] and new["required"]: + text = "now required" + if new["req_default"]: + text += ", req_default: %s" % new["req_default"] + fieldprint(old, new, "", text, reprs) + elif attr == "stored": + if old[attr] != new[attr]: + if new["stored"]: + text = "is now stored" + else: + text = "not stored anymore" + fieldprint(old, new, "", text, reprs) + elif attr == "isfunction": + if old[attr] != new[attr]: + if new["isfunction"]: + text = "now a function" + else: + text = "not a function anymore" + fieldprint(old, new, "", text, reprs) + elif attr == "isproperty": + if old[attr] != new[attr]: + if new[attr]: + text = "now a property" + else: + text = "not a property anymore" + fieldprint(old, new, "", text, reprs) + elif attr == "isrelated": + if old[attr] != new[attr]: + if new[attr]: + text = "now related" + else: + text = "not related anymore" + fieldprint(old, new, "", text, reprs) + elif old[attr] != new[attr]: + fieldprint(old, new, attr, "", reprs) + + +def compare_sets(old_records, new_records): + """ + Compare a set of OpenUpgrade field representations. + Try to match the equivalent fields in both sets. + Return a textual representation of changes in a dictionary with + module names as keys. Special case is the 'general' key + which contains overall remarks and matching statistics. + """ + reprs = collections.defaultdict(list) + + def clean_records(records): + result = [] + for record in records: + if record["field"] not in IGNORE_FIELDS: + result.append(record) + return result + + old_records = clean_records(old_records) + new_records = clean_records(new_records) + + origlen = len(old_records) + new_models = {column["model"] for column in new_records} + old_models = {column["model"] for column in old_records} + + matched_direct = 0 + matched_other_module = 0 + matched_other_type = 0 + in_obsolete_models = 0 + + obsolete_models = [] + for model in old_models: + if model not in new_models: + if model_map(model) not in new_models: + obsolete_models.append(model) + + non_obsolete_old_records = [] + for column in copy.copy(old_records): + if column["model"] in obsolete_models: + in_obsolete_models += 1 + else: + non_obsolete_old_records.append(column) + + def match(match_fields, report_fields, warn=False): + count = 0 + for column in copy.copy(non_obsolete_old_records): + found = search(column, new_records, match_fields) + if found: + if warn: + pass + # print "Tentatively" + report_generic(found, column, report_fields, reprs) + old_records.remove(column) + non_obsolete_old_records.remove(column) + new_records.remove(found) + count += 1 + return count + + matched_direct = match( + ["module", "mode", "model", "field"], + [ + "relation", + "type", + "selection_keys", + "inherits", + "stored", + "isfunction", + "isrelated", + "required", + "table", + ], + ) + + # other module, same type and operation + matched_other_module = match( + ["mode", "model", "field", "type"], + [ + "module", + "relation", + "selection_keys", + "inherits", + "stored", + "isfunction", + "isrelated", + "required", + "table", + ], + ) + + # other module, same operation, other type + matched_other_type = match( + ["mode", "model", "field"], + [ + "relation", + "type", + "selection_keys", + "inherits", + "stored", + "isfunction", + "isrelated", + "required", + "table", + ], + ) + + printkeys = [ + "relation", + "required", + "selection_keys", + "req_default", + "inherits", + "mode", + "attachment", + ] + for column in old_records: + # we do not care about removed non stored function fields + if not column["stored"] and (column["isfunction"] or column["isrelated"]): + continue + if column["mode"] == "create": + column["mode"] = "" + extra_message = ", ".join( + [ + k + ": " + str(column[k]) if k != str(column[k]) else k + for k in printkeys + if column[k] + ] + ) + if extra_message: + extra_message = " " + extra_message + fieldprint(column, "", "", "DEL" + extra_message, reprs) + + printkeys.extend( + [ + "hasdefault", + ] + ) + for column in new_records: + # we do not care about newly added non stored function fields + if not column["stored"] and (column["isfunction"] or column["isrelated"]): + continue + if column["mode"] == "create": + column["mode"] = "" + printkeys_plus = printkeys.copy() + if column["isfunction"] or column["isrelated"]: + printkeys_plus.extend(["isfunction", "isrelated", "stored"]) + extra_message = ", ".join( + [ + k + ": " + str(column[k]) if k != str(column[k]) else k + for k in printkeys_plus + if column[k] + ] + ) + if extra_message: + extra_message = " " + extra_message + fieldprint(column, "", "", "NEW" + extra_message, reprs) + + for line in [ + "# %d fields matched," % (origlen - len(old_records)), + "# Direct match: %d" % matched_direct, + "# Found in other module: %d" % matched_other_module, + "# Found with different type: %d" % matched_other_type, + "# In obsolete models: %d" % in_obsolete_models, + "# Not matched: %d" % len(old_records), + "# New columns: %d" % len(new_records), + ]: + reprs["general"].append(line) + return reprs + + +def compare_xml_sets(old_records, new_records): + reprs = collections.defaultdict(list) + + def match(match_fields, match_type="direct"): + matched_records = [] + for column in copy.copy(old_records): + found = search(column, new_records, match_fields) + if found: + old_records.remove(column) + new_records.remove(found) + if match_type != "direct": + column["old"] = True + found["new"] = True + column[match_type] = found["module"] + found[match_type] = column["module"] + found["domain"] = ( + column["domain"] != found["domain"] + and column["domain"] != "[]" + and found["domain"] is False + ) + column["domain"] = False + column["noupdate_switched"] = False + found["noupdate_switched"] = column["noupdate"] != found["noupdate"] + if match_type != "direct": + matched_records.append(column) + matched_records.append(found) + elif (match_type == "direct" and found["domain"]) or found[ + "noupdate_switched" + ]: + matched_records.append(found) + return matched_records + + # direct match + modified_records = match(["module", "model", "name"]) + + # other module, same full xmlid + moved_records = match(["model", "name"], "moved") + + # other module, same suffix, other prefix + renamed_records = match(["model", "suffix", "other_prefix"], "renamed") + + for record in old_records: + record["old"] = True + record["domain"] = False + record["noupdate_switched"] = False + for record in new_records: + record["new"] = True + record["domain"] = False + record["noupdate_switched"] = False + + sorted_records = sorted( + old_records + new_records + moved_records + renamed_records + modified_records, + key=lambda k: (k["model"], "old" in k, k["name"]), + ) + for entry in sorted_records: + content = "" + if "old" in entry: + content = "DEL %(model)s: %(name)s" % entry + if "moved" in entry: + content += " [potentially moved to %(moved)s module]" % entry + elif "renamed" in entry: + content += " [renamed to %(renamed)s module]" % entry + elif "new" in entry: + content = "NEW %(model)s: %(name)s" % entry + if "moved" in entry: + content += " [potentially moved from %(moved)s module]" % entry + elif "renamed" in entry: + content += " [renamed from %(renamed)s module]" % entry + if "old" not in entry and "new" not in entry: + content = "%(model)s: %(name)s" % entry + if entry["domain"]: + content += " (deleted domain)" + if entry["noupdate"]: + content += " (noupdate)" + if entry["noupdate_switched"]: + content += " (noupdate switched)" + reprs[module_map(entry["module"])].append(content) + return reprs + + +def compare_model_sets(old_records, new_records): + """ + Compare a set of OpenUpgrade model representations. + """ + reprs = collections.defaultdict(list) + + new_models = {column["model"]: column["module"] for column in new_records} + old_models = {column["model"]: column["module"] for column in old_records} + + obsolete_models = [] + for column in copy.copy(old_records): + model = column["model"] + if model in old_models: + if model not in new_models: + if model_map(model) not in new_models: + obsolete_models.append(model) + text = "obsolete model %s" % model + if column["model_type"]: + text += " [%s]" % column["model_type"] + reprs[module_map(column["module"])].append(text) + reprs["general"].append( + "obsolete model %s [module %s]" + % (model, module_map(column["module"])) + ) + else: + moved_module = "" + if module_map(column["module"]) != new_models[model_map(model)]: + moved_module = " in module %s" % new_models[model_map(model)] + text = "obsolete model {} (renamed to {}{})".format( + model, + model_map(model), + moved_module, + ) + if column["model_type"]: + text += " [%s]" % column["model_type"] + reprs[module_map(column["module"])].append(text) + reprs["general"].append( + "obsolete model %s (renamed to %s) [module %s]" + % (model, model_map(model), module_map(column["module"])) + ) + else: + if module_map(column["module"]) != new_models[model]: + text = "model {} (moved to {})".format(model, new_models[model]) + if column["model_type"]: + text += " [%s]" % column["model_type"] + reprs[module_map(column["module"])].append(text) + text = "model {} (moved from {})".format(model, old_models[model]) + if column["model_type"]: + text += " [%s]" % column["model_type"] + + for column in copy.copy(new_records): + model = column["model"] + if model in new_models: + if model not in old_models: + if inv_model_map(model) not in old_models: + text = "new model %s" % model + if column["model_type"]: + text += " [%s]" % column["model_type"] + reprs[column["module"]].append(text) + reprs["general"].append( + "new model {} [module {}]".format(model, column["module"]) + ) + else: + moved_module = "" + if column["module"] != module_map(old_models[inv_model_map(model)]): + moved_module = ( + " in module %s" % old_models[inv_model_map(model)] + ) + text = "new model {} (renamed from {}{})".format( + model, + inv_model_map(model), + moved_module, + ) + if column["model_type"]: + text += " [%s]" % column["model_type"] + reprs[column["module"]].append(text) + reprs["general"].append( + "new model %s (renamed from %s) [module %s]" + % (model, inv_model_map(model), column["module"]) + ) + else: + if column["module"] != module_map(old_models[model]): + text = "model {} (moved from {})".format(model, old_models[model]) + if column["model_type"]: + text += " [%s]" % column["model_type"] + reprs[column["module"]].append(text) + return reprs diff --git a/openupgrade_records/models/__init__.py b/openupgrade_records/models/__init__.py new file mode 100644 index 000000000000..e48a5671f376 --- /dev/null +++ b/openupgrade_records/models/__init__.py @@ -0,0 +1,3 @@ +from . import openupgrade_attribute +from . import openupgrade_comparison_config +from . import openupgrade_record diff --git a/openupgrade_records/models/openupgrade_attribute.py b/openupgrade_records/models/openupgrade_attribute.py new file mode 100644 index 000000000000..a05bce63ffc8 --- /dev/null +++ b/openupgrade_records/models/openupgrade_attribute.py @@ -0,0 +1,18 @@ +# Copyright 2011-2015 Therp BV +# Copyright 2016 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo import fields, models + + +class OpenupgradeAttribute(models.Model): + _name = "openupgrade.attribute" + _description = "OpenUpgrade Attribute" + + name = fields.Char(readonly=True) + value = fields.Char(readonly=True) + record_id = fields.Many2one( + "openupgrade.record", + ondelete="CASCADE", + readonly=True, + ) diff --git a/openupgrade_records/models/openupgrade_comparison_config.py b/openupgrade_records/models/openupgrade_comparison_config.py new file mode 100644 index 000000000000..70af8cb6bbc2 --- /dev/null +++ b/openupgrade_records/models/openupgrade_comparison_config.py @@ -0,0 +1,82 @@ +# Copyright 2011-2015 Therp BV +# Copyright 2016 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +import logging + +from odoo import fields, models +from odoo.exceptions import UserError +from odoo.tools.translate import _ + +from .. import apriori + + +class OpenupgradeComparisonConfig(models.Model): + _name = "openupgrade.comparison.config" + _description = "OpenUpgrade Comparison Configuration" + + name = fields.Char() + server = fields.Char(required=True, default="localhost") + port = fields.Integer(required=True, default=8069) + database = fields.Char(required=True) + username = fields.Char(required=True, default="admin") + password = fields.Char(required=True, default="admin") + last_log = fields.Text() + + def get_connection(self): + self.ensure_one() + import odoorpc + + remote = odoorpc.ODOO(self.server, port=self.port) + remote.login(self.database, self.username, self.password) + return remote + + def test_connection(self): + self.ensure_one() + try: + connection = self.get_connection() + user_model = connection.env["res.users"] + ids = user_model.search([("login", "=", "admin")]) + user_info = user_model.read([ids[0]], ["name"])[0] + except Exception as e: + raise UserError(_("Connection failed.\n\nDETAIL: %s") % e) + raise UserError(_("%s is connected.") % user_info["name"]) + + def analyze(self): + """ Run the analysis wizard """ + self.ensure_one() + wizard = self.env["openupgrade.analysis.wizard"].create( + {"server_config": self.id} + ) + return { + "name": wizard._description, + "view_mode": "form", + "res_model": wizard._name, + "type": "ir.actions.act_window", + "target": "new", + "res_id": wizard.id, + "nodestroy": True, + } + + def install_modules(self): + """ Install same modules as in source DB """ + self.ensure_one() + connection = self.get_connection() + remote_module_obj = connection.env["ir.module.module"] + remote_module_ids = remote_module_obj.search([("state", "=", "installed")]) + + modules = [] + for module_id in remote_module_ids: + mod = remote_module_obj.read([module_id], ["name"])[0] + mod_name = mod["name"] + mod_name = apriori.renamed_modules.get(mod_name, mod_name) + modules.append(mod_name) + _logger = logging.getLogger(__name__) + _logger.debug("remote modules %s", modules) + local_modules = self.env["ir.module.module"].search( + [("name", "in", modules), ("state", "=", "uninstalled")] + ) + _logger.debug("local modules %s", ",".join(local_modules.mapped("name"))) + if local_modules: + local_modules.write({"state": "to install"}) + return {} diff --git a/openupgrade_records/models/openupgrade_record.py b/openupgrade_records/models/openupgrade_record.py new file mode 100644 index 000000000000..3ae2a664f0da --- /dev/null +++ b/openupgrade_records/models/openupgrade_record.py @@ -0,0 +1,99 @@ +# Copyright 2011-2015 Therp BV +# Copyright 2016 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo import api, fields, models + + +class OpenupgradeRecord(models.Model): + _name = "openupgrade.record" + _description = "OpenUpgrade Record" + + name = fields.Char(readonly=True) + module = fields.Char(readonly=True) + model = fields.Char(readonly=True) + field = fields.Char(readonly=True) + mode = fields.Selection( + [("create", "Create"), ("modify", "Modify")], + help="Set to Create if a field is newly created " + "in this module. If this module modifies an attribute of an " + "existing field, set to Modify.", + readonly=True, + ) + type = fields.Selection( # Uh oh, reserved keyword + [("field", "Field"), ("xmlid", "XML ID"), ("model", "Model")], readonly=True + ) + attribute_ids = fields.One2many("openupgrade.attribute", "record_id", readonly=True) + noupdate = fields.Boolean(readonly=True) + domain = fields.Char(readonly=True) + prefix = fields.Char(compute="_compute_prefix_and_suffix") + suffix = fields.Char(compute="_compute_prefix_and_suffix") + model_original_module = fields.Char(compute="_compute_model_original_module") + model_type = fields.Char(compute="_compute_model_type") + + @api.depends("name") + def _compute_prefix_and_suffix(self): + for rec in self: + rec.prefix, rec.suffix = rec.name.split(".", 1) + + @api.depends("model", "type") + def _compute_model_original_module(self): + for rec in self: + if rec.type == "model": + rec.model_original_module = self.env[rec.model]._original_module + else: + rec.model_original_module = "" + + @api.depends("model", "type") + def _compute_model_type(self): + for rec in self: + if rec.type == "model": + model = self.env[rec.model] + if model._auto and model._transient: + rec.model_type = "transient" + elif model._auto: + rec.model_type = "" + elif not model._auto and model._abstract: + rec.model_type = "abstract" + else: + rec.model_type = "sql_view" + else: + rec.model_type = "" + + @api.model + def field_dump(self): + keys = [ + "attachment", + "module", + "mode", + "model", + "field", + "type", + "isfunction", + "isproperty", + "isrelated", + "relation", + "required", + "stored", + "selection_keys", + "req_default", + "hasdefault", + "table", + "inherits", + ] + + template = {x: False for x in keys} + data = [] + for record in self.search([("type", "=", "field")]): + repre = template.copy() + repre.update( + { + "module": record.module, + "model": record.model, + "field": record.field, + "mode": record.mode, + } + ) + repre.update({x.name: x.value for x in record.attribute_ids}) + data.append(repre) + return data diff --git a/openupgrade_records/readme/CONTRIBUTORS.rst b/openupgrade_records/readme/CONTRIBUTORS.rst new file mode 100644 index 000000000000..5de0e93e5453 --- /dev/null +++ b/openupgrade_records/readme/CONTRIBUTORS.rst @@ -0,0 +1,7 @@ +* Stefan Rijnhart +* Holger Brunn +* Pedro M. Baeza +* Ferdinand Gassauer +* Florent Xicluna +* Miquel Raïch +* Sylvain LE GAL diff --git a/openupgrade_records/readme/DESCRIPTION.rst b/openupgrade_records/readme/DESCRIPTION.rst new file mode 100644 index 000000000000..ff1a57292d42 --- /dev/null +++ b/openupgrade_records/readme/DESCRIPTION.rst @@ -0,0 +1 @@ +This module provides the tool to generate the database analysis files that indicate how the Odoo data model and module data have changed between two versions of Odoo. Database analysis files for the core modules are included in the OpenUpgrade distribution so as a migration script developer you will not usually need to use this tool yourself. If you do need to run your analysis of a custom set of modules, please refer to the documentation here: https://doc.therp.nl/openupgrade/analysis.html diff --git a/openupgrade_records/readme/INSTALL.rst b/openupgrade_records/readme/INSTALL.rst new file mode 100644 index 000000000000..3e8727a20cfd --- /dev/null +++ b/openupgrade_records/readme/INSTALL.rst @@ -0,0 +1,4 @@ +.. code-block:: shell + + pip install odoorpc==0.7.0 + pip install git+https://github.com/OCA/openupgradelib.git@master diff --git a/openupgrade_records/readme/ROADMAP.rst b/openupgrade_records/readme/ROADMAP.rst new file mode 100644 index 000000000000..734c22444065 --- /dev/null +++ b/openupgrade_records/readme/ROADMAP.rst @@ -0,0 +1,3 @@ +* scripts/compare_noupdate_xml_records.py should be integrated in the analysis process (#590) +* Log removed modules in the module that owned them (#468) +* Detect renamed many2many tables (#213) diff --git a/openupgrade_records/readme/USAGE.rst b/openupgrade_records/readme/USAGE.rst new file mode 100644 index 000000000000..1333ed77b7e1 --- /dev/null +++ b/openupgrade_records/readme/USAGE.rst @@ -0,0 +1 @@ +TODO diff --git a/openupgrade_records/security/ir.model.access.csv b/openupgrade_records/security/ir.model.access.csv new file mode 100644 index 000000000000..f3e1a6e532e0 --- /dev/null +++ b/openupgrade_records/security/ir.model.access.csv @@ -0,0 +1,7 @@ +"id","name","model_id:id","group_id:id","perm_read","perm_write","perm_create","perm_unlink" +"access_openupgrade_record","openupgrade.record all","model_openupgrade_record",,1,0,0,0 +"access_openupgrade_attribute","openupgrade.attribute all","model_openupgrade_attribute",,1,0,0,0 +"access_openupgrade_comparison_config","openupgrade.comparison.config","model_openupgrade_comparison_config",base.group_system,1,1,1,1 +access_openupgrade_analysis_wizard,access_openupgrade_analysis_wizard,model_openupgrade_analysis_wizard,base.group_system,1,1,1,1 +access_openupgrade_generate_records_wizard,access_openupgrade_generate_records_wizard,model_openupgrade_generate_records_wizard,base.group_system,1,1,1,1 +access_openupgrade_install_all_wizard,access_openupgrade_install_all_wizard,model_openupgrade_install_all_wizard,base.group_system,1,1,1,1 diff --git a/openupgrade_records/views/menu.xml b/openupgrade_records/views/menu.xml new file mode 100644 index 000000000000..b49d51a5409d --- /dev/null +++ b/openupgrade_records/views/menu.xml @@ -0,0 +1,12 @@ + + + + + + + diff --git a/openupgrade_records/views/openupgrade_comparison_config.xml b/openupgrade_records/views/openupgrade_comparison_config.xml new file mode 100644 index 000000000000..c6905c0d6602 --- /dev/null +++ b/openupgrade_records/views/openupgrade_comparison_config.xml @@ -0,0 +1,73 @@ + + + + + openupgrade.comparison.config + + + + + + + + + + + + openupgrade.comparison.config + +
+ + + + + + + + +