diff --git a/upgrade_analysis/__init__.py b/upgrade_analysis/__init__.py index 7fc3dd81d5c..ca02278dfe6 100644 --- a/upgrade_analysis/__init__.py +++ b/upgrade_analysis/__init__.py @@ -4,5 +4,4 @@ from . import blacklist from . import apriori from . import compare -from . import upgrade_loading from . import upgrade_log diff --git a/upgrade_analysis/models/upgrade_analysis.py b/upgrade_analysis/models/upgrade_analysis.py index a579696fa1b..52eff5d85af 100644 --- a/upgrade_analysis/models/upgrade_analysis.py +++ b/upgrade_analysis/models/upgrade_analysis.py @@ -3,6 +3,7 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). # flake8: noqa: C901 +import logging import os from odoo import fields, models @@ -11,6 +12,7 @@ from .. import compare +_logger = logging.getLogger(__name__) _IGNORE_MODULES = ["openupgrade_records", "upgrade_analysis"] @@ -31,6 +33,10 @@ class UpgradeAnalysis(models.Model): ) log = fields.Text(readonly=True) + upgrade_path = fields.Char( + default=config.get("upgrade_path", False), + help="The base file path to save the analyse files of Odoo modules", + ) write_files = fields.Boolean( help="Write analysis files to the module directories", default=True @@ -50,10 +56,12 @@ def _write_file( ): module = self.env["ir.module.module"].search([("name", "=", module_name)])[0] if module.is_odoo_module: - upgrade_path = config.get("upgrade_path", False) - if not upgrade_path: - return "ERROR: could not find 'upgrade_path' config:\n" - module_path = os.path.join(upgrade_path, module_name) + if not self.upgrade_path: + return ( + "ERROR: no upgrade_path set when writing analysis of %s\n" + % module_name + ) + module_path = os.path.join(self.upgrade_path, module_name) else: module_path = get_module_path(module_name) if not module_path: @@ -71,6 +79,7 @@ def _write_file( f = open(logfile, "w") except Exception: return "ERROR: could not open file %s for writing:\n" % logfile + _logger.debug("Writing analysis to %s", logfile) f.write(content) f.close() return None diff --git a/upgrade_analysis/odoo_patch/__init__.py b/upgrade_analysis/odoo_patch/__init__.py index 56a70dbc176..4a183795658 100644 --- a/upgrade_analysis/odoo_patch/__init__.py +++ b/upgrade_analysis/odoo_patch/__init__.py @@ -1,2 +1,3 @@ -from . import odoo from . import addons +from . import odoo +from . import odoo_patch diff --git a/upgrade_analysis/odoo_patch/addons/__init__.py b/upgrade_analysis/odoo_patch/addons/__init__.py index e5aa886bacb..3247dc7b6b5 100644 --- a/upgrade_analysis/odoo_patch/addons/__init__.py +++ b/upgrade_analysis/odoo_patch/addons/__init__.py @@ -1,3 +1,3 @@ from . import mrp -from . import stock from . import point_of_sale +from . import stock diff --git a/upgrade_analysis/odoo_patch/addons/mrp/__init__.py b/upgrade_analysis/odoo_patch/addons/mrp/__init__.py index 925f56ed264..94f30010f58 100644 --- a/upgrade_analysis/odoo_patch/addons/mrp/__init__.py +++ b/upgrade_analysis/odoo_patch/addons/mrp/__init__.py @@ -1,19 +1,10 @@ from odoo.addons import mrp +from ...odoo_patch import OdooPatch -def _pre_init_mrp(cr): - """ Allow installing MRP in databases with large stock.move table (>1M records) - - Creating the computed+stored field stock_move.is_done is terribly slow with the ORM and - leads to "Out of Memory" crashes - """ - # - # don't try to add 'is_done' column, because it will fail - # when executing the generation of records, in the openupgrade_records - # module. - # cr.execute("""ALTER TABLE "stock_move" ADD COLUMN "is_done" bool;""") - # cr.execute("""UPDATE stock_move - # SET is_done=COALESCE(state in ('done', 'cancel'), FALSE);""") - # +class PreInitHookPatch(OdooPatch): + target = mrp + method_names = ['_pre_init_mrp'] - -mrp._pre_init_mrp = _pre_init_mrp + def _pre_init_mrp(cr): + """ Don't try to create an existing column on reinstall """ diff --git a/upgrade_analysis/odoo_patch/addons/point_of_sale/__init__.py b/upgrade_analysis/odoo_patch/addons/point_of_sale/__init__.py deleted file mode 100644 index 0650744f6bc..00000000000 --- a/upgrade_analysis/odoo_patch/addons/point_of_sale/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import models diff --git a/upgrade_analysis/odoo_patch/addons/point_of_sale/models/__init__.py b/upgrade_analysis/odoo_patch/addons/point_of_sale/models/__init__.py deleted file mode 100644 index db8634ade1f..00000000000 --- a/upgrade_analysis/odoo_patch/addons/point_of_sale/models/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import pos_config diff --git a/upgrade_analysis/odoo_patch/addons/point_of_sale/models/pos_config.py b/upgrade_analysis/odoo_patch/addons/point_of_sale/models/pos_config.py deleted file mode 100644 index ac0f5dc5a49..00000000000 --- a/upgrade_analysis/odoo_patch/addons/point_of_sale/models/pos_config.py +++ /dev/null @@ -1,21 +0,0 @@ -from odoo import api -from odoo.addons.point_of_sale.models.pos_config import PosConfig - -if True: - - @api.model - def post_install_pos_localisation(self, companies=False): - # - # don't try to setup_defaults, because it will fail - # when executing the generation of records, in the openupgrade_records - # module. - # self = self.sudo() - # if not companies: - # companies = self.env['res.company'].search([]) - # for company in companies.filtered('chart_template_id'): - # pos_configs = self.search([('company_id', '=', company.id)]) - # pos_configs.setup_defaults(company) - pass - # - -PosConfig.post_install_pos_localisation = post_install_pos_localisation diff --git a/upgrade_analysis/odoo_patch/addons/stock/__init__.py b/upgrade_analysis/odoo_patch/addons/stock/__init__.py index b66d7f484cb..a28a083cea8 100644 --- a/upgrade_analysis/odoo_patch/addons/stock/__init__.py +++ b/upgrade_analysis/odoo_patch/addons/stock/__init__.py @@ -1,17 +1,10 @@ from odoo.addons import stock +from ...odoo_patch import OdooPatch -def pre_init_hook(cr): - # - # don't uninstall data as this breaks the analysis - # Origin of this code is https://github.com/odoo/odoo/issues/22243 - # env = api.Environment(cr, SUPERUSER_ID, {}) - # env['ir.model.data'].search([ - # ('model', 'like', '%stock%'), - # ('module', '=', 'stock') - # ]).unlink() - pass - # +class PreInitHookPatch(OdooPatch): + target = stock + method_names = ['pre_init_hook'] - -stock.pre_init_hook = pre_init_hook + def pre_init_hook(cr): + """ Don't unlink stock data on reinstall """ diff --git a/upgrade_analysis/odoo_patch/odoo/__init__.py b/upgrade_analysis/odoo_patch/odoo/__init__.py index 5629ec98c48..4541924d99f 100644 --- a/upgrade_analysis/odoo_patch/odoo/__init__.py +++ b/upgrade_analysis/odoo_patch/odoo/__init__.py @@ -1,5 +1,3 @@ +from . import models from . import modules -from . import service from . import tools -from . import http -from . import models diff --git a/upgrade_analysis/odoo_patch/odoo/http.py b/upgrade_analysis/odoo_patch/odoo/http.py deleted file mode 100644 index e11c558fb90..00000000000 --- a/upgrade_analysis/odoo_patch/odoo/http.py +++ /dev/null @@ -1,32 +0,0 @@ -# flake8: noqa -# pylint: skip-file - -import odoo -from odoo.service import security -from odoo.http import SessionExpiredException, request, OpenERPSession - -if True: - def _check_security(self): - """ - Check the current authentication parameters to know if those are still - valid. This method should be called at each request. If the - authentication fails, a :exc:`SessionExpiredException` is raised. - """ - if not self.db or not self.uid: - raise SessionExpiredException("Session expired") - # We create our own environment instead of the request's one. - # to avoid creating it without the uid since request.uid isn't set yet - env = odoo.api.Environment(request.cr, self.uid, self.context) - # here we check if the session is still valid - if not security.check_session(self, env): - # - # When asking openupgrade_records to generate records - # over jsonrpc, a query on res_users in the call above locks this - # table for the sql operations that are triggered by the - # reinstallation of the base module - env.cr.rollback() - # - raise SessionExpiredException("Session expired") - - -OpenERPSession.check_security = _check_security diff --git a/upgrade_analysis/odoo_patch/odoo/models.py b/upgrade_analysis/odoo_patch/odoo/models.py index 201f3f4fe63..09c10c41836 100644 --- a/upgrade_analysis/odoo_patch/odoo/models.py +++ b/upgrade_analysis/odoo_patch/odoo/models.py @@ -1,179 +1,21 @@ -# flake8: noqa -# pylint: skip-file - -import odoo -import psycopg2 -from odoo import _ -from odoo.models import fix_import_export_id_paths, BaseModel, _logger +from odoo import api, models +from ..odoo_patch import OdooPatch from ... import upgrade_log -if True: - def _load(self, fields, data): - """ - Attempts to load the data matrix, and returns a list of ids (or - ``False`` if there was an error and no id could be generated) and a - list of messages. - - The ids are those of the records created and saved (in database), in - the same order they were extracted from the file. They can be passed - directly to :meth:`~read` - - :param fields: list of fields to import, at the same index as the corresponding data - :type fields: list(str) - :param data: row-major matrix of data to import - :type data: list(list(str)) - :returns: {ids: list(int)|False, messages: [Message][, lastrow: int]} - """ - self.flush() - - # determine values of mode, current_module and noupdate - mode = self._context.get('mode', 'init') - current_module = self._context.get('module', '__import__') - noupdate = self._context.get('noupdate', False) - # add current module in context for the conversion of xml ids - self = self.with_context(_import_current_module=current_module) - - cr = self._cr - cr.execute('SAVEPOINT model_load') - - fields = [fix_import_export_id_paths(f) for f in fields] - fg = self.fields_get() - - ids = [] - messages = [] - ModelData = self.env['ir.model.data'] - - # list of (xid, vals, info) for records to be created in batch - batch = [] - batch_xml_ids = set() - # models in which we may have created / modified data, therefore might - # require flushing in order to name_search: the root model and any - # o2m - creatable_models = {self._name} - for field_path in fields: - if field_path[0] in (None, 'id', '.id'): - continue - model_fields = self._fields - if isinstance(model_fields[field_path[0]], odoo.fields.Many2one): - # this only applies for toplevel m2o (?) fields - if field_path[0] in (self.env.context.get('name_create_enabled_fieds') or {}): - creatable_models.add(model_fields[field_path[0]].comodel_name) - for field_name in field_path: - if field_name in (None, 'id', '.id'): - break - - if isinstance(model_fields[field_name], odoo.fields.One2many): - comodel = model_fields[field_name].comodel_name - creatable_models.add(comodel) - model_fields = self.env[comodel]._fields - - def flush(*, xml_id=None, model=None): - if not batch: - return +class BaseModelPatch(OdooPatch): + target = models.BaseModel + method_names = ['_convert_records'] - assert not (xml_id and model), \ - "flush can specify *either* an external id or a model, not both" - - if xml_id and xml_id not in batch_xml_ids: - if xml_id not in self.env: - return - if model and model not in creatable_models: - return - - data_list = [ - dict(xml_id=xid, values=vals, info=info, noupdate=noupdate) - for xid, vals, info in batch - ] - batch.clear() - batch_xml_ids.clear() - - # try to create in batch - try: - with cr.savepoint(): - recs = self._load_records(data_list, mode == 'update') - ids.extend(recs.ids) - return - except psycopg2.InternalError as e: - # broken transaction, exit and hope the source error was already logged - if not any(message['type'] == 'error' for message in messages): - info = data_list[0]['info'] - messages.append(dict(info, type='error', message=_(u"Unknown database error: '%s'", e))) - return - except Exception: - pass - - errors = 0 - # try again, this time record by record - for i, rec_data in enumerate(data_list, 1): - try: - with cr.savepoint(): - rec = self._load_records([rec_data], mode == 'update') - ids.append(rec.id) - except psycopg2.Warning as e: - info = rec_data['info'] - messages.append(dict(info, type='warning', message=str(e))) - except psycopg2.Error as e: - info = rec_data['info'] - messages.append(dict(info, type='error', **PGERROR_TO_OE[e.pgcode](self, fg, info, e))) - # Failed to write, log to messages, rollback savepoint (to - # avoid broken transaction) and keep going - errors += 1 - except Exception as e: - _logger.debug("Error while loading record", exc_info=True) - info = rec_data['info'] - message = (_(u'Unknown error during import:') + u' %s: %s' % (type(e), e)) - moreinfo = _('Resolve other errors first') - messages.append(dict(info, type='error', message=message, moreinfo=moreinfo)) - # Failed for some reason, perhaps due to invalid data supplied, - # rollback savepoint and keep going - errors += 1 - if errors >= 10 and (errors >= i / 10): - messages.append({ - 'type': 'warning', - 'message': _(u"Found more than 10 errors and more than one error per 10 records, interrupted to avoid showing too many errors.") - }) - break - - # make 'flush' available to the methods below, in the case where XMLID - # resolution fails, for instance - flush_self = self.with_context(import_flush=flush) - - # TODO: break load's API instead of smuggling via context? - limit = self._context.get('_import_limit') - if limit is None: - limit = float('inf') - extracted = flush_self._extract_records(fields, data, log=messages.append, limit=limit) - - converted = flush_self._convert_records(extracted, log=messages.append) - - info = {'rows': {'to': -1}} - for id, xid, record, info in converted: + @api.model + def _convert_records(self, records, log=lambda a: None): + """ Log data ids that are imported with `load` """ + current_module = self.env.context['module'] + for res in BaseModelPatch._convert_records._original_method( + self, records, log=log): + _id, xid, _record, _info = res if xid: xid = xid if '.' in xid else "%s.%s" % (current_module, xid) - batch_xml_ids.add(xid) - # - # log csv records upgrade_log.log_xml_id(self.env.cr, current_module, xid) - # - elif id: - record['id'] = id - batch.append((xid, record, info)) - - flush() - if any(message['type'] == 'error' for message in messages): - cr.execute('ROLLBACK TO SAVEPOINT model_load') - ids = False - # cancel all changes done to the registry/ormcache - self.pool.reset_changes() - - nextrow = info['rows']['to'] + 1 - if nextrow < limit: - nextrow = 0 - return { - 'ids': ids, - 'messages': messages, - 'nextrow': nextrow, - } -BaseModel.load = _load + yield res diff --git a/upgrade_analysis/odoo_patch/odoo/modules/__init__.py b/upgrade_analysis/odoo_patch/odoo/modules/__init__.py index 90de5b4ff4e..7246323f930 100644 --- a/upgrade_analysis/odoo_patch/odoo/modules/__init__.py +++ b/upgrade_analysis/odoo_patch/odoo/modules/__init__.py @@ -1,12 +1 @@ -# Minor changes. (call to safe_eval changed) -# otherwise : adapted to V14 -from . import graph - -# A lot of changes in the core functions. -from . import loading - -# Adapted to V14 -from . import migration - -# Adapted to V14 from . import registry diff --git a/upgrade_analysis/odoo_patch/odoo/modules/graph.py b/upgrade_analysis/odoo_patch/odoo/modules/graph.py deleted file mode 100644 index b0bedef3ea6..00000000000 --- a/upgrade_analysis/odoo_patch/odoo/modules/graph.py +++ /dev/null @@ -1,108 +0,0 @@ -# flake8: noqa -# pylint: skip-file - -import logging -import odoo -import odoo.tools as tools -from odoo.tools.safe_eval import safe_eval - -from odoo.modules.graph import Graph - -_logger = logging.getLogger(__name__) - - -if True: - - def _update_from_db(self, cr): - if not len(self): - return - # update the graph with values from the database (if exist) - ## First, we set the default values for each package in graph - additional_data = {key: {'id': 0, 'state': 'uninstalled', 'dbdemo': False, 'installed_version': None} for key in self.keys()} - ## Then we get the values from the database - cr.execute('SELECT name, id, state, demo AS dbdemo, latest_version AS installed_version' - ' FROM ir_module_module' - ' WHERE name IN %s',(tuple(additional_data),) - ) - - ## and we update the default values with values from the database - additional_data.update((x['name'], x) for x in cr.dictfetchall()) - - # - # Prevent reloading of demo data from the new version on major upgrade - if ('base' in self and additional_data['base']['dbdemo'] and - additional_data['base']['installed_version'] < - odoo.release.major_version): - cr.execute("UPDATE ir_module_module SET demo = false") - for data in additional_data.values(): - data['dbdemo'] = False - # - - for package in self.values(): - for k, v in additional_data[package.name].items(): - setattr(package, k, v) - - - def _add_modules(self, cr, module_list, force=None): - if force is None: - force = [] - packages = [] - len_graph = len(self) - - # - # force additional dependencies for the upgrade process if given - # in config file - forced_deps = tools.config.get_misc('openupgrade', 'force_deps', '{}') - forced_deps = tools.config.get_misc('openupgrade', - 'force_deps_' + odoo.release.version, - forced_deps) - forced_deps = safe_eval(forced_deps) - # - - for module in module_list: - # This will raise an exception if no/unreadable descriptor file. - # NOTE The call to load_information_from_description_file is already - # done by db.initialize, so it is possible to not do it again here. - info = odoo.modules.module.load_information_from_description_file(module) - if info and info['installable']: - # - info['depends'].extend(forced_deps.get(module, [])) - # - packages.append((module, info)) # TODO directly a dict, like in get_modules_with_version - elif module != 'studio_customization': - _logger.warning('module %s: not installable, skipped', module) - - dependencies = dict([(p, info['depends']) for p, info in packages]) - current, later = set([p for p, info in packages]), set() - - while packages and current > later: - package, info = packages[0] - deps = info['depends'] - - # if all dependencies of 'package' are already in the graph, add 'package' in the graph - if all(dep in self for dep in deps): - if not package in current: - packages.pop(0) - continue - later.clear() - current.remove(package) - node = self.add_node(package, info) - for kind in ('init', 'demo', 'update'): - if package in tools.config[kind] or 'all' in tools.config[kind] or kind in force: - setattr(node, kind, True) - else: - later.add(package) - packages.append((package, info)) - packages.pop(0) - - self.update_from_db(cr) - - for package in later: - unmet_deps = [p for p in dependencies[package] if p not in self] - _logger.info('module %s: Unmet dependencies: %s', package, ', '.join(unmet_deps)) - - return len(self) - len_graph - - -Graph.update_from_db = _update_from_db -Graph.add_modules = _add_modules diff --git a/upgrade_analysis/odoo_patch/odoo/modules/loading.py b/upgrade_analysis/odoo_patch/odoo/modules/loading.py deleted file mode 100644 index 4194fdc626d..00000000000 --- a/upgrade_analysis/odoo_patch/odoo/modules/loading.py +++ /dev/null @@ -1,556 +0,0 @@ -# flake8: noqa -# pylint: skip-file - -import itertools -import logging -import sys -import time - -import odoo -import odoo.tools as tools -from odoo import api, SUPERUSER_ID -from odoo.modules import loading -from odoo.modules.module import adapt_version, load_openerp_module, initialize_sys_path - -from odoo.modules.loading import load_data, load_demo, _check_module_names -from .... import upgrade_loading - -import os - -_logger = logging.getLogger(__name__) -_test_logger = logging.getLogger('odoo.tests') - - -def _load_module_graph(cr, graph, status=None, perform_checks=True, - skip_modules=None, report=None, models_to_check=None, upg_registry=None): - # - """Migrates+Updates or Installs all module nodes from ``graph`` - :param graph: graph of module nodes to load - :param status: deprecated parameter, unused, left to avoid changing signature in 8.0 - :param perform_checks: whether module descriptors should be checked for validity (prints warnings - for same cases) - :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped - :return: list of modules that were installed or updated - """ - if skip_modules is None: - skip_modules = [] - - if models_to_check is None: - models_to_check = set() - - processed_modules = [] - loaded_modules = [] - registry = odoo.registry(cr.dbname) - migrations = odoo.modules.migration.MigrationManager(cr, graph) - module_count = len(graph) - _logger.info('loading %d modules...', module_count) - - # - # suppress commits to have the upgrade of one module in just one transaction - cr.commit_org = cr.commit - cr.commit = lambda *args: None - cr.rollback_org = cr.rollback - cr.rollback = lambda *args: None - # - - # register, instantiate and initialize models for each modules - t0 = time.time() - loading_extra_query_count = odoo.sql_db.sql_counter - loading_cursor_query_count = cr.sql_log_count - - models_updated = set() - - for index, package in enumerate(graph, 1): - module_name = package.name - module_id = package.id - - # - if module_name in skip_modules or module_name in loaded_modules: - # - continue - - module_t0 = time.time() - module_cursor_query_count = cr.sql_log_count - module_extra_query_count = odoo.sql_db.sql_counter - - needs_update = ( - hasattr(package, "init") - or hasattr(package, "update") - or package.state in ("to install", "to upgrade") - ) - module_log_level = logging.DEBUG - if needs_update: - module_log_level = logging.INFO - _logger.log(module_log_level, 'Loading module %s (%d/%d)', module_name, index, module_count) - - if needs_update: - if package.name != 'base': - registry.setup_models(cr) - migrations.migrate_module(package, 'pre') - if package.name != 'base': - env = api.Environment(cr, SUPERUSER_ID, {}) - env['base'].flush() - - load_openerp_module(package.name) - - new_install = package.state == 'to install' - if new_install: - py_module = sys.modules['odoo.addons.%s' % (module_name,)] - pre_init = package.info.get('pre_init_hook') - if pre_init: - getattr(py_module, pre_init)(cr) - - model_names = registry.load(cr, package) - - mode = 'update' - if hasattr(package, 'init') or package.state == 'to install': - mode = 'init' - - loaded_modules.append(package.name) - if needs_update: - models_updated |= set(model_names) - models_to_check -= set(model_names) - registry.setup_models(cr) - # - # rebuild the local registry based on the loaded models - local_registry = {} - env = api.Environment(cr, SUPERUSER_ID, {}) - for model in env.values(): - if not model._auto: - continue - upgrade_loading.log_model(model, local_registry) - upgrade_loading.compare_registries( - cr, package.name, upg_registry, local_registry) - # - - registry.init_models(cr, model_names, {'module': package.name}, new_install) - elif package.state != 'to remove': - # The current module has simply been loaded. The models extended by this module - # and for which we updated the schema, must have their schema checked again. - # This is because the extension may have changed the model, - # e.g. adding required=True to an existing field, but the schema has not been - # updated by this module because it's not marked as 'to upgrade/to install'. - models_to_check |= set(model_names) & models_updated - - idref = {} - - if needs_update: - env = api.Environment(cr, SUPERUSER_ID, {}) - # Can't put this line out of the loop: ir.module.module will be - # registered by init_models() above. - module = env['ir.module.module'].browse(module_id) - - if perform_checks: - module._check() - - if package.state == 'to upgrade': - # upgrading the module information - module.write(module.get_values_from_terp(package.data)) - load_data(cr, idref, mode, kind='data', package=package) - demo_loaded = package.dbdemo = load_demo(cr, package, idref, mode) - cr.execute('update ir_module_module set demo=%s where id=%s', (demo_loaded, module_id)) - module.invalidate_cache(['demo']) - - # - # add 'try' block for logging exceptions - # as errors in post scripts seem to be dropped - try: - migrations.migrate_module(package, 'post') - except Exception as exc: - _logger.error('Error executing post migration script for module %s: %s', - package, exc) - raise - # - - # Update translations for all installed languages - overwrite = odoo.tools.config["overwrite_existing_translations"] - module.with_context(overwrite=overwrite)._update_translations() - - if package.name is not None: - registry._init_modules.add(package.name) - - if needs_update: - if new_install: - post_init = package.info.get('post_init_hook') - if post_init: - getattr(py_module, post_init)(cr, registry) - - if mode == 'update': - # validate the views that have not been checked yet - env['ir.ui.view']._validate_module_views(module_name) - - # need to commit any modification the module's installation or - # update made to the schema or data so the tests can run - # (separately in their own transaction) - # - # commit after processing every module as well, for - # easier debugging and continuing an interrupted migration - cr.commit_org() - # - # run tests - if os.environ.get('OPENUPGRADE_TESTS') and package.name is not None: - prefix = '.migrations' - registry.openupgrade_test_prefixes[package.name] = prefix - report.record_result(odoo.modules.module.run_unit_tests(module_name, openupgrade_prefix=prefix)) - # - # commit module_n state and version immediatly - # to avoid invalid database state if module_n+1 raises an - # exception - cr.commit_org() - # - - package.load_state = package.state - package.load_version = package.installed_version - package.state = 'installed' - for kind in ('init', 'demo', 'update'): - if hasattr(package, kind): - delattr(package, kind) - module.flush() - - extra_queries = odoo.sql_db.sql_counter - module_extra_query_count - test_queries - extras = [] - if test_queries: - extras.append(f'+{test_queries} test') - if extra_queries: - extras.append(f'+{extra_queries} other') - _logger.log( - module_log_level, "Module %s loaded in %.2fs%s, %s queries%s", - module_name, time.time() - module_t0, - f' (incl. {test_time:.2f}s test)' if test_time else '', - cr.sql_log_count - module_cursor_query_count, - f' ({", ".join(extras)})' if extras else '' - ) - if test_results and not test_results.wasSuccessful(): - _logger.error( - "Module %s: %d failures, %d errors of %d tests", - module_name, len(test_results.failures), len(test_results.errors), - test_results.testsRun - ) - - _logger.runbot("%s modules loaded in %.2fs, %s queries (+%s extra)", - len(graph), - time.time() - t0, - cr.sql_log_count - loading_cursor_query_count, - odoo.sql_db.sql_counter - loading_extra_query_count) # extra queries: testes, notify, any other closed cursor - - # - # restore commit method - cr.commit = cr.commit_org - cr.commit() - # - - return loaded_modules, processed_modules - - -def _load_marked_modules(cr, graph, states, force, progressdict, report, - loaded_modules, perform_checks, models_to_check=None, upg_registry=None): - # - """Loads modules marked with ``states``, adding them to ``graph`` and - ``loaded_modules`` and returns a list of installed/upgraded modules.""" - - if models_to_check is None: - models_to_check = set() - - processed_modules = [] - while True: - cr.execute("SELECT name from ir_module_module WHERE state IN %s" ,(tuple(states),)) - module_list = [name for (name,) in cr.fetchall() if name not in graph] - # - module_list = upgrade_loading.add_module_dependencies(cr, module_list) - # - if not module_list: - break - graph.add_modules(cr, module_list, force) - _logger.debug('Updating graph with %d more modules', len(module_list)) - # - # add upg_registry - loaded, processed = _load_module_graph( - cr, graph, progressdict, report=report, skip_modules=loaded_modules, - perform_checks=perform_checks, models_to_check=models_to_check, - upg_registry=upg_registry, - ) - # - processed_modules.extend(processed) - loaded_modules.extend(loaded) - if not processed: - break - return processed_modules - - -def _load_modules(db, force_demo=False, status=None, update_module=False): - initialize_sys_path() - - force = [] - if force_demo: - force.append('demo') - - # - upg_registry = {} - # - - models_to_check = set() - - with db.cursor() as cr: - if not odoo.modules.db.is_initialized(cr): - if not update_module: - _logger.error("Database %s not initialized, you can force it with `-i base`", cr.dbname) - return - _logger.info("init db") - odoo.modules.db.initialize(cr) - update_module = True # process auto-installed modules - tools.config["init"]["all"] = 1 - if not tools.config['without_demo']: - tools.config["demo"]['all'] = 1 - - # This is a brand new registry, just created in - # odoo.modules.registry.Registry.new(). - registry = odoo.registry(cr.dbname) - - if 'base' in tools.config['update'] or 'all' in tools.config['update']: - cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed')) - - # STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps) - graph = odoo.modules.graph.Graph() - graph.add_module(cr, 'base', force) - if not graph: - _logger.critical('module base cannot be loaded! (hint: verify addons-path)') - raise ImportError('Module `base` cannot be loaded! (hint: verify addons-path)') - - # processed_modules: for cleanup step after install - # loaded_modules: to avoid double loading - report = registry._assertion_report - # - # add upg_registry - loaded_modules, processed_modules = _load_module_graph( - cr, graph, status, perform_checks=update_module, - report=report, models_to_check=models_to_check, upg_registry=upg_registry) - - # - load_lang = tools.config.pop('load_language') - if load_lang or update_module: - # some base models are used below, so make sure they are set up - registry.setup_models(cr) - - if load_lang: - for lang in load_lang.split(','): - tools.load_language(cr, lang) - - # STEP 2: Mark other modules to be loaded/updated - if update_module: - env = api.Environment(cr, SUPERUSER_ID, {}) - Module = env['ir.module.module'] - _logger.info('updating modules list') - Module.update_list() - - _check_module_names(cr, itertools.chain(tools.config['init'], tools.config['update'])) - - module_names = [k for k, v in tools.config['init'].items() if v] - if module_names: - modules = Module.search([('state', '=', 'uninstalled'), ('name', 'in', module_names)]) - if modules: - modules.button_install() - - module_names = [k for k, v in tools.config['update'].items() if v] - if module_names: - # - # in standard Odoo, '--update all' just means: - # '--update base + upward (installed) dependencies. This breaks - # the chain when new glue modules are encountered. - # E.g. purchase in 8.0 depends on stock_account and report, - # both of which are new. They may be installed, but purchase as - # an upward dependency is not selected for upgrade. - # Therefore, explicitely select all installed modules for - # upgrading in OpenUpgrade in that case. - domain = [('state', '=', 'installed')] - if 'all' not in module_names: - domain.append(('name', 'in', module_names)) - modules = Module.search(domain) - # - if modules: - modules.button_upgrade() - - cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base')) - Module.invalidate_cache(['state']) - Module.flush() - - # STEP 3: Load marked modules (skipping base which was done in STEP 1) - # IMPORTANT: this is done in two parts, first loading all installed or - # partially installed modules (i.e. installed/to upgrade), to - # offer a consistent system to the second part: installing - # newly selected modules. - # We include the modules 'to remove' in the first step, because - # they are part of the "currently installed" modules. They will - # be dropped in STEP 6 later, before restarting the loading - # process. - # IMPORTANT 2: We have to loop here until all relevant modules have been - # processed, because in some rare cases the dependencies have - # changed, and modules that depend on an uninstalled module - # will not be processed on the first pass. - # It's especially useful for migrations. - previously_processed = -1 - while previously_processed < len(processed_modules): - previously_processed = len(processed_modules) - # - # add upg_registry - processed_modules += _load_marked_modules(cr, graph, - ['installed', 'to upgrade', 'to remove'], - force, status, report, loaded_modules, update_module, models_to_check, upg_registry) - # - if update_module: - # - # add upg_registry - processed_modules += _load_marked_modules(cr, graph, - ['to install'], force, status, report, - loaded_modules, update_module, models_to_check, upg_registry) - # - # check that new module dependencies have been properly installed after a migration/upgrade - cr.execute("SELECT name from ir_module_module WHERE state IN ('to install', 'to upgrade')") - module_list = [name for (name,) in cr.fetchall()] - if module_list: - _logger.error("Some modules have inconsistent states, some dependencies may be missing: %s", sorted(module_list)) - - # check that all installed modules have been loaded by the registry after a migration/upgrade - cr.execute("SELECT name from ir_module_module WHERE state = 'installed' and name != 'studio_customization'") - module_list = [name for (name,) in cr.fetchall() if name not in graph] - if module_list: - _logger.error("Some modules are not loaded, some dependencies or manifest may be missing: %s", sorted(module_list)) - - registry.loaded = True - registry.setup_models(cr) - - # STEP 3.5: execute migration end-scripts - migrations = odoo.modules.migration.MigrationManager(cr, graph) - for package in graph: - migrations.migrate_module(package, 'end') - - # STEP 3.6: apply remaining constraints in case of an upgrade - registry.finalize_constraints() - - # STEP 4: Finish and cleanup installations - if processed_modules: - env = api.Environment(cr, SUPERUSER_ID, {}) - cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""") - for (model, name) in cr.fetchall(): - if model in registry and not registry[model]._abstract: - _logger.warning('The model %s has no access rules, consider adding one. E.g. access_%s,access_%s,model_%s,base.group_user,1,0,0,0', - model, model.replace('.', '_'), model.replace('.', '_'), model.replace('.', '_')) - - cr.execute("SELECT model from ir_model") - for (model,) in cr.fetchall(): - if model in registry: - env[model]._check_removed_columns(log=True) - elif _logger.isEnabledFor(logging.INFO): # more an info that a warning... - _logger.runbot("Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)", model) - - # Cleanup orphan records - env['ir.model.data']._process_end(processed_modules) - env['base'].flush() - - for kind in ('init', 'demo', 'update'): - tools.config[kind] = {} - - # STEP 5: Uninstall modules to remove - if update_module: - # Remove records referenced from ir_model_data for modules to be - # removed (and removed the references from ir_model_data). - cr.execute("SELECT name, id FROM ir_module_module WHERE state=%s", ('to remove',)) - modules_to_remove = dict(cr.fetchall()) - if modules_to_remove: - env = api.Environment(cr, SUPERUSER_ID, {}) - pkgs = reversed([p for p in graph if p.name in modules_to_remove]) - for pkg in pkgs: - uninstall_hook = pkg.info.get('uninstall_hook') - if uninstall_hook: - py_module = sys.modules['odoo.addons.%s' % (pkg.name,)] - getattr(py_module, uninstall_hook)(cr, registry) - - Module = env['ir.module.module'] - Module.browse(modules_to_remove.values()).module_uninstall() - # Recursive reload, should only happen once, because there should be no - # modules to remove next time - cr.commit() - _logger.info('Reloading registry once more after uninstalling modules') - api.Environment.reset() - registry = odoo.modules.registry.Registry.new( - cr.dbname, force_demo, status, update_module - ) - registry.check_tables_exist(cr) - cr.commit() - return registry - - # STEP 5.5: Verify extended fields on every model - # This will fix the schema of all models in a situation such as: - # - module A is loaded and defines model M; - # - module B is installed/upgraded and extends model M; - # - module C is loaded and extends model M; - # - module B and C depend on A but not on each other; - # The changes introduced by module C are not taken into account by the upgrade of B. - if models_to_check: - registry.init_models(cr, list(models_to_check), {'models_to_check': True}) - - # STEP 6: verify custom views on every model - if update_module: - env = api.Environment(cr, SUPERUSER_ID, {}) - env['res.groups']._update_user_groups_view() - View = env['ir.ui.view'] - for model in registry: - try: - View._validate_custom_views(model) - except Exception as e: - _logger.warning('invalid custom view(s) for model %s: %s', model, tools.ustr(e)) - - if report.wasSuccessful(): - _logger.info('Modules loaded.') - else: - _logger.error('At least one test failed when loading the modules.') - - # STEP 8: call _register_hook on every model - # This is done *exactly once* when the registry is being loaded. See the - # management of those hooks in `Registry.setup_models`: all the calls to - # setup_models() done here do not mess up with hooks, as registry.ready - # is False. - env = api.Environment(cr, SUPERUSER_ID, {}) - for model in env.values(): - model._register_hook() - env['base'].flush() - - # STEP 9: save installed/updated modules for post-install tests - registry.updated_modules += processed_modules - -loading.load_module_graph = _load_module_graph -loading.load_marked_modules = _load_marked_modules -loading.load_modules = _load_modules -odoo.modules.load_modules = _load_modules diff --git a/upgrade_analysis/odoo_patch/odoo/modules/migration.py b/upgrade_analysis/odoo_patch/odoo/modules/migration.py deleted file mode 100644 index 0346c2b8c55..00000000000 --- a/upgrade_analysis/odoo_patch/odoo/modules/migration.py +++ /dev/null @@ -1,118 +0,0 @@ -# flake8: noqa -# pylint: skip-file - -import logging -import os -from os.path import join as opj -import odoo.release as release -from odoo.tools.parse_version import parse_version - -import odoo -from odoo.modules.migration import load_script -from odoo.modules import migration - -_logger = logging.getLogger(__name__) - - -if True: - def _migrate_module(self, pkg, stage): - assert stage in ('pre', 'post', 'end') - stageformat = { - 'pre': '[>%s]', - 'post': '[%s>]', - 'end': '[$%s]', - } - state = pkg.state if stage in ('pre', 'post') else getattr(pkg, 'load_state', None) - - # - # In openupgrade, also run migration scripts upon installation. - # We want to always pass in pre and post migration files and use a new - # argument in the migrate decorator (explained in the docstring) - # to decide if we want to do something if a new module is installed - # during the migration. - if not (hasattr(pkg, 'update') or state in ('to upgrade', 'to install')): - # - return - - def convert_version(version): - if version.count('.') >= 2: - return version # the version number already containt the server version - return "%s.%s" % (release.major_version, version) - - def _get_migration_versions(pkg, stage): - versions = sorted({ - ver - for lv in self.migrations[pkg.name].values() - for ver, lf in lv.items() - if lf - }, key=lambda k: parse_version(convert_version(k))) - if "0.0.0" in versions: - # reorder versions - versions.remove("0.0.0") - if stage == "pre": - versions.insert(0, "0.0.0") - else: - versions.append("0.0.0") - return versions - - def _get_migration_files(pkg, version, stage): - """ return a list of migration script files - """ - m = self.migrations[pkg.name] - lst = [] - - mapping = { - 'module': opj(pkg.name, 'migrations'), - 'module_upgrades': opj(pkg.name, 'upgrades'), - } - - for path in odoo.upgrade.__path__: - if os.path.exists(opj(path, pkg.name)): - mapping['upgrade'] = opj(path, pkg.name) - break - - for x in mapping: - if version in m.get(x): - for f in m[x][version]: - if not f.startswith(stage + '-'): - continue - lst.append(opj(mapping[x], version, f)) - lst.sort() - return lst - - installed_version = getattr(pkg, 'load_version', pkg.installed_version) or '' - parsed_installed_version = parse_version(installed_version) - current_version = parse_version(convert_version(pkg.data['version'])) - - versions = _get_migration_versions(pkg, stage) - - for version in versions: - if ((version == "0.0.0" and parsed_installed_version < current_version) - or parsed_installed_version < parse_version(convert_version(version)) <= current_version): - - strfmt = {'addon': pkg.name, - 'stage': stage, - 'version': stageformat[stage] % version, - } - - for pyfile in _get_migration_files(pkg, version, stage): - name, ext = os.path.splitext(os.path.basename(pyfile)) - if ext.lower() != '.py': - continue - mod = None - try: - mod = load_script(pyfile, name) - _logger.info('module %(addon)s: Running migration %(version)s %(name)s' % dict(strfmt, name=mod.__name__)) - migrate = mod.migrate - except ImportError: - _logger.exception('module %(addon)s: Unable to load %(stage)s-migration file %(file)s' % dict(strfmt, file=pyfile)) - raise - except AttributeError: - _logger.error('module %(addon)s: Each %(stage)s-migration file must have a "migrate(cr, installed_version)" function' % strfmt) - else: - migrate(self.cr, installed_version) - finally: - if mod: - del mod - -migration.migrate_module = _migrate_module diff --git a/upgrade_analysis/odoo_patch/odoo/modules/registry.py b/upgrade_analysis/odoo_patch/odoo/modules/registry.py index 4c5f50d4e71..3d0981a001e 100644 --- a/upgrade_analysis/odoo_patch/odoo/modules/registry.py +++ b/upgrade_analysis/odoo_patch/odoo/modules/registry.py @@ -1,58 +1,29 @@ -# flake8: noqa -# pylint: skip-file - -from collections import deque -from contextlib import closing -import odoo -from odoo.tools.lru import LRU - -from odoo.modules import registry - - -if True: - - def _init(self, db_name): - self.models = {} # model name/model instance mapping - self._sql_constraints = set() - self._init = True - self._assertion_report = odoo.tests.runner.OdooTestResult() - self._fields_by_model = None - self._ordinary_tables = None - self._constraint_queue = deque() - self.__cache = LRU(8192) - - # modules fully loaded (maintained during init phase by `loading` module) - self._init_modules = set() - self.updated_modules = [] # installed/updated modules - # - self.openupgrade_test_prefixes = {} - # - self.loaded_xmlids = set() - - self.db_name = db_name - self._db = odoo.sql_db.db_connect(db_name) - - # cursor for test mode; None means "normal" mode - self.test_cr = None - self.test_lock = None - - # Indicates that the registry is - self.loaded = False # whether all modules are loaded - self.ready = False # whether everything is set up - - # Inter-process signaling: - # The `base_registry_signaling` sequence indicates the whole registry - # must be reloaded. - # The `base_cache_signaling sequence` indicates all caches must be - # invalidated (i.e. cleared). - self.registry_sequence = None - self.cache_sequence = None - - # Flags indicating invalidation of the registry or the cache. - self.registry_invalidated = False - self.cache_invalidated = False - - with closing(self.cursor()) as cr: - self.has_unaccent = odoo.modules.db.has_unaccent(cr) - -registry.init = _init +import logging +from threading import current_thread +from odoo import api, SUPERUSER_ID +from ...odoo_patch import OdooPatch +from .... import upgrade_log +from odoo.modules.registry import Registry + +_logger = logging.getLogger(__name__) + + +class RegistryPatch(OdooPatch): + target = Registry + method_names = ['init_models'] + + def init_models(self, cr, model_names, context, install=True): + module_name = context['module'] + _logger.debug('Logging models of module %s', module_name) + upg_registry = current_thread()._upgrade_registry + local_registry = {} + env = api.Environment(cr, SUPERUSER_ID, {}) + for model in env.values(): + if not model._auto: + continue + upgrade_log.log_model(model, local_registry) + upgrade_log.compare_registries( + cr, context['module'], upg_registry, local_registry) + + return RegistryPatch.init_models._original_method( + self, cr, model_names, context, install=install) diff --git a/upgrade_analysis/odoo_patch/odoo/service/__init__.py b/upgrade_analysis/odoo_patch/odoo/service/__init__.py deleted file mode 100644 index a96314d0f68..00000000000 --- a/upgrade_analysis/odoo_patch/odoo/service/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Import disabled, because the function run_unit_tests() -# disappeared in V14. -# TODO: OpenUpgrade Core maintainers : FIXME. -# from . import server diff --git a/upgrade_analysis/odoo_patch/odoo/service/server.py b/upgrade_analysis/odoo_patch/odoo/service/server.py deleted file mode 100644 index a2a998e69df..00000000000 --- a/upgrade_analysis/odoo_patch/odoo/service/server.py +++ /dev/null @@ -1,71 +0,0 @@ -# flake8: noqa -# pylint: skip-file - -import logging -import os -import time - -import odoo -from odoo.tools import config -from odoo.modules.registry import Registry - -from odoo.service import server -from odoo.service.server import load_test_file_py - -_logger = logging.getLogger(__name__) - - -def preload_registries(dbnames): - """ Preload a registries, possibly run a test file.""" - # TODO: move all config checks to args dont check tools.config here - dbnames = dbnames or [] - rc = 0 - for dbname in dbnames: - try: - update_module = config['init'] or config['update'] - registry = Registry.new(dbname, update_module=update_module) - - # run test_file if provided - if config['test_file']: - test_file = config['test_file'] - if not os.path.isfile(test_file): - _logger.warning('test file %s cannot be found', test_file) - elif not test_file.endswith('py'): - _logger.warning('test file %s is not a python file', test_file) - else: - _logger.info('loading test file %s', test_file) - with odoo.api.Environment.manage(): - load_test_file_py(registry, test_file) - - # run post-install tests - if config['test_enable']: - t0 = time.time() - t0_sql = odoo.sql_db.sql_counter - module_names = (registry.updated_modules if update_module else - sorted(registry._init_modules)) - _logger.info("Starting post tests") - tests_before = registry._assertion_report.testsRun - with odoo.api.Environment.manage(): - for module_name in module_names: - result = loader.run_suite(loader.make_suite(module_name, 'post_install'), module_name) - registry._assertion_report.update(result) - # - # run deferred unit tests - for module_name, prefix in registry.openupgrade_test_prefixes: - result = run_unit_tests(module_name, position='post_install', openupgrade_prefix=prefix) - registry._assertion_report.record_result(result) - # - _logger.info("%d post-tests in %.2fs, %s queries", - registry._assertion_report.testsRun - tests_before, - time.time() - t0, - odoo.sql_db.sql_counter - t0_sql) - - if not registry._assertion_report.wasSuccessful(): - rc += 1 - except Exception: - _logger.critical('Failed to initialize database `%s`.', dbname, exc_info=True) - return -1 - return rc - - -server.preload_registries = preload_registries diff --git a/upgrade_analysis/odoo_patch/odoo/tools/__init__.py b/upgrade_analysis/odoo_patch/odoo/tools/__init__.py index 6ad156515dc..99a9527ec5a 100644 --- a/upgrade_analysis/odoo_patch/odoo/tools/__init__.py +++ b/upgrade_analysis/odoo_patch/odoo/tools/__init__.py @@ -1,2 +1 @@ from . import convert -from . import view_validation diff --git a/upgrade_analysis/odoo_patch/odoo/tools/convert.py b/upgrade_analysis/odoo_patch/odoo/tools/convert.py index 8c1f710a52e..7130859d628 100644 --- a/upgrade_analysis/odoo_patch/odoo/tools/convert.py +++ b/upgrade_analysis/odoo_patch/odoo/tools/convert.py @@ -1,23 +1,13 @@ -# flake8: noqa -# pylint: skip-file - +from ...odoo_patch import OdooPatch from .... import upgrade_log - from odoo.tools.convert import xml_import -if True: - def __test_xml_id(self, xml_id): - if '.' in xml_id: - module, id = xml_id.split('.', 1) - assert '.' not in id, """The ID reference "%s" must contain -maximum one dot. They are used to refer to other modules ID, in the -form: module.record_id""" % (xml_id,) - if module != self.module: - modcnt = self.env['ir.module.module'].search_count([('name', '=', module), ('state', '=', 'installed')]) - assert modcnt == 1, """The ID "%s" refers to an uninstalled module""" % (xml_id,) +class XMLImportPatch(OdooPatch): + target = xml_import + method_names = ['_test_xml_id'] - # OpenUpgrade: log entry of XML imports + def _test_xml_id(self, xml_id): + res = XMLImportPatch._test_xml_id._original_method(self, xml_id) upgrade_log.log_xml_id(self.env.cr, self.module, xml_id) - -xml_import._test_xml_id = __test_xml_id + return res diff --git a/upgrade_analysis/odoo_patch/odoo/tools/view_validation.py b/upgrade_analysis/odoo_patch/odoo/tools/view_validation.py deleted file mode 100644 index e6c8243241a..00000000000 --- a/upgrade_analysis/odoo_patch/odoo/tools/view_validation.py +++ /dev/null @@ -1,29 +0,0 @@ -# flake8: noqa -# pylint: skip-file - -# from odoo.addons.openupgrade_framework.openupgrade import openupgrade_log - -from odoo.tools import view_validation -from odoo.tools.view_validation import _validators, _logger - - -def _valid_view(arch, **kwargs): - for pred in _validators[arch.tag]: - # - # Do not raise blocking error, because it's normal to - # have inconsistent views in an openupgrade process - check = pred(arch, **kwargs) or 'Warning' - # - if not check: - _logger.error("Invalid XML: %s", pred.__doc__) - return False - if check == "Warning": - # - # Don't show this warning as useless and too much verbose - # _logger.warning("Invalid XML: %s", pred.__doc__) - # - return "Warning" - return True - - -view_validation.valid_view = _valid_view diff --git a/upgrade_analysis/odoo_patch/odoo_patch.py b/upgrade_analysis/odoo_patch/odoo_patch.py new file mode 100644 index 00000000000..ac9f02fee2e --- /dev/null +++ b/upgrade_analysis/odoo_patch/odoo_patch.py @@ -0,0 +1,59 @@ +import logging + +_logger = logging.getLogger(__name__) + + +class OdooPatch(object): + """ Simple mechanism to apply a collection of monkeypatches using a + context manager. + + Classes can register their monkeypatches by inheriting from this class. + They need to define a `target` member, referring to the object or module + that needs to be patched, and a list `method_names`. They also need to + redefine those methods under the same name. + + The original method is made available on the new method as + `_original_method`. + + Example: + + ``` + from odoo import api + from odoo.addons.some_module.models.my_model import MyModel + + class MyModelPatch(OdooPatch): + target = MyModel + method_names = ['do_something'] + + @api.model + def do_something(self): + res = MyModelPatch.do_something._original_method() + ... + return res + ``` + + Usage: + + ``` + with OdooPatch(): + do_something() + ``` + """ + def __enter__(self): + for cls in OdooPatch.__subclasses__(): + for method_name in cls.method_names: + method = getattr(cls, method_name) + setattr(method, '_original_method', + getattr(cls.target, method_name)) + setattr(cls.target, method_name, method) + + def __exit__(self, exc_type, exc_value, tb): + for cls in OdooPatch.__subclasses__(): + for method_name in cls.method_names: + method = getattr(cls.target, method_name) + if hasattr(method, '_original_method'): + setattr(cls.target, method_name, method._original_method) + else: + _logger.warn( + '_original_method not found on method %s of class %s', + method_name, cls.target) diff --git a/upgrade_analysis/upgrade_loading.py b/upgrade_analysis/upgrade_loading.py deleted file mode 100644 index 1187a5900f4..00000000000 --- a/upgrade_analysis/upgrade_loading.py +++ /dev/null @@ -1,316 +0,0 @@ -# Copyright 2011-2015 Therp BV -# Copyright 2016-2019 Opener B.V. -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). - -# flake8: noqa: C901 - -import logging - -from openupgradelib.openupgrade_tools import table_exists - -from odoo import release -from odoo.modules.module import get_module_path -from odoo.tools.config import config -from odoo.tools.safe_eval import safe_eval - -# A collection of functions used in -# odoo/modules/loading.py - -_logger = logging.getLogger(__name__) - - -def add_module_dependencies(cr, module_list): - """ - Select (new) dependencies from the modules in the list - so that we can inject them into the graph at upgrade - time. Used in the modified OpenUpgrade Server, - not to be called from migration scripts - - Also take the OpenUpgrade configuration directives 'forced_deps' - and 'autoinstall' into account. From any additional modules - that these directives can add, the dependencies are added as - well (but these directives are not checked for the occurrence - of any of the dependencies). - """ - if not module_list: - return module_list - - modules_in = list(module_list) - forced_deps = safe_eval( - config.get_misc( - "openupgrade", - "forced_deps_" + release.version, - config.get_misc("openupgrade", "forced_deps", "{}"), - ) - ) - - autoinstall = safe_eval( - config.get_misc( - "openupgrade", - "autoinstall_" + release.version, - config.get_misc("openupgrade", "autoinstall", "{}"), - ) - ) - - for module in list(module_list): - module_list += forced_deps.get(module, []) - module_list += autoinstall.get(module, []) - - module_list = list(set(module_list)) - - dependencies = module_list - while dependencies: - cr.execute( - """ - SELECT DISTINCT dep.name - FROM - ir_module_module, - ir_module_module_dependency dep - WHERE - module_id = ir_module_module.id - AND ir_module_module.name in %s - AND dep.name not in %s - """, - ( - tuple(dependencies), - tuple(module_list), - ), - ) - - dependencies = [x[0] for x in cr.fetchall()] - module_list += dependencies - - # Select auto_install modules of which all dependencies - # are fulfilled based on the modules we know are to be - # installed - cr.execute( - """ - SELECT name from ir_module_module WHERE state IN %s - """, - (("installed", "to install", "to upgrade"),), - ) - modules = list(set(module_list + [row[0] for row in cr.fetchall()])) - cr.execute( - """ - SELECT name from ir_module_module m - WHERE auto_install IS TRUE - AND state = 'uninstalled' - AND NOT EXISTS( - SELECT id FROM ir_module_module_dependency d - WHERE d.module_id = m.id - AND name NOT IN %s) - """, - (tuple(modules),), - ) - auto_modules = [row[0] for row in cr.fetchall() if get_module_path(row[0])] - if auto_modules: - _logger.info("Selecting autoinstallable modules %s", ",".join(auto_modules)) - module_list += auto_modules - - # Set proper state for new dependencies so that any init scripts are run - cr.execute( - """ - UPDATE ir_module_module SET state = 'to install' - WHERE name IN %s AND name NOT IN %s AND state = 'uninstalled' - """, - (tuple(module_list), tuple(modules_in)), - ) - return module_list - - -def log_model(model, local_registry): - """ - OpenUpgrade: Store the characteristics of the BaseModel and its fields - in the local registry, so that we can compare changes with the - main registry - """ - - if not model._name: - return - - typemap = {"monetary": "float"} - - # Deferred import to prevent import loop - from odoo import models - - # persistent models only - if isinstance(model, models.TransientModel): - return - - def isfunction(model, k): - if ( - model._fields[k].compute - and not model._fields[k].related - and not model._fields[k].company_dependent - ): - return "function" - return "" - - def isproperty(model, k): - if model._fields[k].company_dependent: - return "property" - return "" - - def isrelated(model, k): - if model._fields[k].related: - return "related" - return "" - - def _get_relation(v): - if v.type in ("many2many", "many2one", "one2many"): - return v.comodel_name - elif v.type == "many2one_reference": - return v.model_field - else: - return "" - - model_registry = local_registry.setdefault(model._name, {}) - if model._inherits: - model_registry["_inherits"] = {"_inherits": str(model._inherits)} - for k, v in model._fields.items(): - properties = { - "type": typemap.get(v.type, v.type), - "isfunction": isfunction(model, k), - "isproperty": isproperty(model, k), - "isrelated": isrelated(model, k), - "relation": _get_relation(v), - "table": v.relation if v.type == "many2many" else "", - "required": v.required and "required" or "", - "stored": v.store and "stored" or "", - "selection_keys": "", - "req_default": "", - "hasdefault": model._fields[k].default and "hasdefault" or "", - "inherits": "", - } - if v.type == "selection": - if isinstance(v.selection, (tuple, list)): - properties["selection_keys"] = str(sorted([x[0] for x in v.selection])) - else: - properties["selection_keys"] = "function" - elif v.type == "binary": - properties["attachment"] = str(getattr(v, "attachment", False)) - default = model._fields[k].default - if v.required and default: - if ( - callable(default) - or isinstance(default, str) - and getattr(model._fields[k], default, False) - and callable(getattr(model._fields[k], default)) - ): - # todo: in OpenERP 5 (and in 6 as well), - # literals are wrapped in a lambda function - properties["req_default"] = "function" - else: - properties["req_default"] = str(default) - for key, value in properties.items(): - if value: - model_registry.setdefault(k, {})[key] = value - - -def get_record_id(cr, module, model, field, mode): - """ - OpenUpgrade: get or create the id from the record table matching - the key parameter values - """ - cr.execute( - "SELECT id FROM upgrade_record " - "WHERE module = %s AND model = %s AND " - "field = %s AND mode = %s AND type = %s", - (module, model, field, mode, "field"), - ) - record = cr.fetchone() - if record: - return record[0] - cr.execute( - "INSERT INTO upgrade_record " - "(module, model, field, mode, type) " - "VALUES (%s, %s, %s, %s, %s)", - (module, model, field, mode, "field"), - ) - cr.execute( - "SELECT id FROM upgrade_record " - "WHERE module = %s AND model = %s AND " - "field = %s AND mode = %s AND type = %s", - (module, model, field, mode, "field"), - ) - return cr.fetchone()[0] - - -def compare_registries(cr, module, registry, local_registry): - """ - OpenUpgrade: Compare the local registry with the global registry, - log any differences and merge the local registry with - the global one. - """ - if not table_exists(cr, "upgrade_record"): - return - for model, flds in local_registry.items(): - registry.setdefault(model, {}) - for field, attributes in flds.items(): - old_field = registry[model].setdefault(field, {}) - mode = old_field and "modify" or "create" - record_id = False - for key, value in attributes.items(): - if key not in old_field or old_field[key] != value: - if not record_id: - record_id = get_record_id(cr, module, model, field, mode) - cr.execute( - "SELECT id FROM upgrade_attribute " - "WHERE name = %s AND value = %s AND " - "record_id = %s", - (key, value, record_id), - ) - if not cr.fetchone(): - cr.execute( - "INSERT INTO upgrade_attribute " - "(name, value, record_id) VALUES (%s, %s, %s)", - (key, value, record_id), - ) - old_field[key] = value - - -def update_field_xmlid(model, field): - """OpenUpgrade edit start: In rare cases, an old module defined a field - on a model that is not defined in another module earlier in the - chain of inheritance. Then we need to assign the ir.model.fields' - xmlid to this other module, otherwise the column would be dropped - when uninstalling the first module. - An example is res.partner#display_name defined in 7.0 by - account_report_company, but now the field belongs to the base - module - Given that we arrive here in order of inheritance, we simply check - if the field's xmlid belongs to a module already loaded, and if not, - update the record with the correct module name.""" - model.env.cr.execute( - "SELECT f.*, d.module, d.id as xmlid_id, d.name as xmlid " - "FROM ir_model_fields f LEFT JOIN ir_model_data d " - "ON f.id=d.res_id and d.model='ir.model.fields' WHERE f.model=%s", - (model._name,), - ) - for rec in model.env.cr.dictfetchall(): - if ( - "module" in model.env.context - and rec["module"] - and rec["name"] in model._fields.keys() - and rec["module"] != model.env.context["module"] - and rec["module"] not in model.env.registry._init_modules - ): - _logger.info( - "Moving XMLID for ir.model.fields record of %s#%s " "from %s to %s", - model._name, - rec["name"], - rec["module"], - model.env.context["module"], - ) - model.env.cr.execute( - "SELECT id FROM ir_model_data WHERE module=%(module)s " - "AND name=%(xmlid)s", - dict(rec, module=model.env.context["module"]), - ) - if model.env.cr.fetchone(): - _logger.info("Aborting, an XMLID for this module already exists.") - continue - model.env.cr.execute( - "UPDATE ir_model_data SET module=%(module)s " "WHERE id=%(xmlid_id)s", - dict(rec, module=model.env.context["module"]), - ) diff --git a/upgrade_analysis/upgrade_log.py b/upgrade_analysis/upgrade_log.py index 2563ecbc637..10c28596b60 100644 --- a/upgrade_analysis/upgrade_log.py +++ b/upgrade_analysis/upgrade_log.py @@ -6,9 +6,161 @@ from openupgradelib.openupgrade_tools import table_exists +from odoo import models + _logger = logging.getLogger(__name__) +def get_record_id(cr, module, model, field, mode): + """ + OpenUpgrade: get or create the id from the record table matching + the key parameter values + """ + cr.execute( + "SELECT id FROM upgrade_record " + "WHERE module = %s AND model = %s AND " + "field = %s AND mode = %s AND type = %s", + (module, model, field, mode, "field"), + ) + record = cr.fetchone() + if record: + return record[0] + cr.execute( + "INSERT INTO upgrade_record " + "(module, model, field, mode, type) " + "VALUES (%s, %s, %s, %s, %s)", + (module, model, field, mode, "field"), + ) + cr.execute( + "SELECT id FROM upgrade_record " + "WHERE module = %s AND model = %s AND " + "field = %s AND mode = %s AND type = %s", + (module, model, field, mode, "field"), + ) + return cr.fetchone()[0] + + +def compare_registries(cr, module, registry, local_registry): + """ + OpenUpgrade: Compare the local registry with the global registry, + log any differences and merge the local registry with + the global one. + """ + if not table_exists(cr, "upgrade_record"): + return + for model, flds in local_registry.items(): + registry.setdefault(model, {}) + for field, attributes in flds.items(): + old_field = registry[model].setdefault(field, {}) + mode = old_field and "modify" or "create" + record_id = False + for key, value in attributes.items(): + if key not in old_field or old_field[key] != value: + if not record_id: + record_id = get_record_id(cr, module, model, field, mode) + cr.execute( + "SELECT id FROM upgrade_attribute " + "WHERE name = %s AND value = %s AND " + "record_id = %s", + (key, value, record_id), + ) + if not cr.fetchone(): + cr.execute( + "INSERT INTO upgrade_attribute " + "(name, value, record_id) VALUES (%s, %s, %s)", + (key, value, record_id), + ) + old_field[key] = value + + +def isfunction(model, k): + if ( + model._fields[k].compute + and not model._fields[k].related + and not model._fields[k].company_dependent + ): + return "function" + return "" + + +def isproperty(model, k): + if model._fields[k].company_dependent: + return "property" + return "" + + +def isrelated(model, k): + if model._fields[k].related: + return "related" + return "" + + +def _get_relation(v): + if v.type in ("many2many", "many2one", "one2many"): + return v.comodel_name + elif v.type == "many2one_reference": + return v.model_field + else: + return "" + + +def log_model(model, local_registry): + """ + OpenUpgrade: Store the characteristics of the BaseModel and its fields + in the local registry, so that we can compare changes with the + main registry + """ + + if not model._name: + return + + typemap = {"monetary": "float"} + + # persistent models only + if isinstance(model, models.TransientModel): + return + + model_registry = local_registry.setdefault(model._name, {}) + if model._inherits: + model_registry["_inherits"] = {"_inherits": str(model._inherits)} + for k, v in model._fields.items(): + properties = { + "type": typemap.get(v.type, v.type), + "isfunction": isfunction(model, k), + "isproperty": isproperty(model, k), + "isrelated": isrelated(model, k), + "relation": _get_relation(v), + "table": v.relation if v.type == "many2many" else "", + "required": v.required and "required" or "", + "stored": v.store and "stored" or "", + "selection_keys": "", + "req_default": "", + "hasdefault": model._fields[k].default and "hasdefault" or "", + "inherits": "", + } + if v.type == "selection": + if isinstance(v.selection, (tuple, list)): + properties["selection_keys"] = str(sorted([x[0] for x in v.selection])) + else: + properties["selection_keys"] = "function" + elif v.type == "binary": + properties["attachment"] = str(getattr(v, "attachment", False)) + default = model._fields[k].default + if v.required and default: + if ( + callable(default) + or isinstance(default, str) + and getattr(model._fields[k], default, False) + and callable(getattr(model._fields[k], default)) + ): + properties["req_default"] = "function" + else: + properties["req_default"] = str(default) + for key, value in properties.items(): + if value: + model_registry.setdefault(k, {})[key] = value + + def log_xml_id(cr, module, xml_id): """ Log xml_ids at load time in the records table. @@ -49,7 +201,7 @@ def log_xml_id(cr, module, xml_id): ) record = cr.fetchone() if not record: - _logger.warning("Cannot find xml_id %s" % xml_id) + _logger.warning("Cannot find xml_id %s", xml_id) return else: cr.execute( diff --git a/upgrade_analysis/views/view_upgrade_analysis.xml b/upgrade_analysis/views/view_upgrade_analysis.xml index 375253ce9b0..c2ce39170b3 100644 --- a/upgrade_analysis/views/view_upgrade_analysis.xml +++ b/upgrade_analysis/views/view_upgrade_analysis.xml @@ -36,6 +36,10 @@ name="write_files" attrs="{'readonly': [('state', '=', 'done')]}" /> + # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +from threading import current_thread from odoo import _, fields, models from odoo.exceptions import UserError from odoo.modules.registry import Registry +from ..odoo_patch.odoo_patch import OdooPatch + class GenerateWizard(models.TransientModel): _name = "upgrade.generate.record.wizard" @@ -14,26 +17,6 @@ class GenerateWizard(models.TransientModel): state = fields.Selection([("draft", "Draft"), ("done", "Done")], default="draft") - # from openupgradelib import openupgrade_tools - # TODO, SLG, make better a patch in odoo_patch - # def quirk_standard_calendar_attendances(self): - # """Introduced in Odoo 13. The reinstallation causes a one2many value - # in [(0, 0, {})] format to be loaded on top of the first load, causing a - # violation of database constraint.""" - # for cal in ("resource_calendar_std_35h", "resource_calendar_std_38h"): - # record = self.env.ref("resource.%s" % cal, False) - # if record: - # record.attendance_ids.unlink() - - # # Truncate the records table - # if openupgrade_tools.table_exists( - # self.env.cr, "upgrade_attribute" - # ) and openupgrade_tools.table_exists(self.env.cr, "upgrade_record"): - # self.env.cr.execute("TRUNCATE upgrade_attribute, upgrade_record;") - - # # Run any quirks - # self.quirk_standard_calendar_attendances() - def generate(self): """Reinitialize all installed modules. Equivalent of running the server with '-d --init all' @@ -56,7 +39,17 @@ def generate(self): {"state": "to install"} ) self.env.cr.commit() # pylint: disable=invalid-commit - Registry.new(self.env.cr.dbname, update_module=True) + + # Patch the registry on the thread + thread = current_thread() + thread._upgrade_registry = {} + + # Regenerate the registry with monkeypatches that log the records + with OdooPatch(): + Registry.new(self.env.cr.dbname, update_module=True) + + # Free the registry + delattr(thread, "_upgrade_registry") # Set domain property self.env.cr.execute( diff --git a/upgrade_analysis/wizards/upgrade_install_wizard.py b/upgrade_analysis/wizards/upgrade_install_wizard.py index d91232867d9..94cc3a7fece 100644 --- a/upgrade_analysis/wizards/upgrade_install_wizard.py +++ b/upgrade_analysis/wizards/upgrade_install_wizard.py @@ -17,6 +17,10 @@ class UpgradeInstallWizard(models.TransientModel): _name = "upgrade.install.wizard" _description = "Upgrade Install Wizard" + name = fields.Char( + default=_description, + help="Workaround for https://github.com/OCA/odoorpc/issues/57", + ) state = fields.Selection( [("draft", "Draft"), ("done", "Done")], readonly=True, default="draft" ) @@ -52,30 +56,38 @@ def _compute_module_qty(self): for wizard in self: wizard.module_qty = len(wizard.module_ids) - def select_odoo_modules(self): + def select_odoo_modules(self, extra_domain=None): self.ensure_one() - modules = self.env["ir.module.module"].search(self._module_ids_domain()) + modules = self.env["ir.module.module"].search( + self._module_ids_domain(extra_domain=extra_domain) + ) modules = modules.filtered(lambda x: x.is_odoo_module) self.module_ids = modules return self.return_same_form_view() - def select_oca_modules(self): + def select_oca_modules(self, extra_domain=None): self.ensure_one() - modules = self.env["ir.module.module"].search(self._module_ids_domain()) + modules = self.env["ir.module.module"].search( + self._module_ids_domain(extra_domain=extra_domain) + ) modules = modules.filtered(lambda x: x.is_oca_module) self.module_ids = modules return self.return_same_form_view() - def select_other_modules(self): + def select_other_modules(self, extra_domain=None): self.ensure_one() - modules = self.env["ir.module.module"].search(self._module_ids_domain()) + modules = self.env["ir.module.module"].search( + self._module_ids_domain(extra_domain=extra_domain) + ) modules = modules.filtered(lambda x: not (x.is_oca_module or x.is_odoo_module)) self.module_ids = modules return self.return_same_form_view() - def select_installable_modules(self): + def select_installable_modules(self, extra_domain=None): self.ensure_one() - self.module_ids = self.env["ir.module.module"].search(self._module_ids_domain()) + self.module_ids = self.env["ir.module.module"].search( + self._module_ids_domain(extra_domain=extra_domain) + ) return self.return_same_form_view() def unselect_modules(self):