diff --git a/requirements.txt b/requirements.txt index 478901d8128..4694712cf5f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ # generated from manifests external_dependencies +dataclasses odoorpc openupgradelib diff --git a/upgrade_analysis/__manifest__.py b/upgrade_analysis/__manifest__.py index a7d9de82bcf..44c755efba3 100644 --- a/upgrade_analysis/__manifest__.py +++ b/upgrade_analysis/__manifest__.py @@ -20,7 +20,7 @@ ], "installable": True, "external_dependencies": { - "python": ["odoorpc", "openupgradelib"], + "python": ["dataclasses", "odoorpc", "openupgradelib"], }, "license": "AGPL-3", } diff --git a/upgrade_analysis/models/upgrade_analysis.py b/upgrade_analysis/models/upgrade_analysis.py index f42ffdff1ca..e9eca7b59a5 100644 --- a/upgrade_analysis/models/upgrade_analysis.py +++ b/upgrade_analysis/models/upgrade_analysis.py @@ -1,17 +1,28 @@ # Copyright 2011-2015 Therp BV -# Copyright 2016 Opener B.V. +# Copyright 2016-2020 Opener B.V. +# Copyright 2019 Eficent +# Copyright 2020 GRAP # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). # flake8: noqa: C901 import logging import os +from copy import deepcopy + +from lxml import etree from odoo import fields, models -from odoo.exceptions import UserError +from odoo.exceptions import UserError, ValidationError from odoo.modules import get_module_path from odoo.tools import config +from odoo.tools.convert import nodeattr2bool from odoo.tools.translate import _ +try: + from odoo.addons.openupgrade_scripts.apriori import renamed_modules +except ImportError: + renamed_modules = {} + from .. import compare _logger = logging.getLogger(__name__) @@ -246,6 +257,13 @@ def analyze(self): general_log, "upgrade_general_log.txt", ) + + try: + self.generate_noupdate_changes() + except Exception as e: + _logger.exception("Error generating noupdate changes: %s" % e) + general_log += "ERROR: error when generating noupdate changes: %s\n" % e + self.write( { "state": "done", @@ -253,3 +271,215 @@ def analyze(self): } ) return True + + @staticmethod + def _get_node_dict(element): + res = {} + if element is None: + return res + for child in element: + if "name" in child.attrib: + key = "./{}[@name='{}']".format(child.tag, child.attrib["name"]) + res[key] = child + return res + + @staticmethod + def _get_node_value(element): + if "eval" in element.attrib.keys(): + return element.attrib["eval"] + if "ref" in element.attrib.keys(): + return element.attrib["ref"] + if not len(element): + return element.text + return etree.tostring(element) + + def _get_xml_diff( + self, remote_update, remote_noupdate, local_update, local_noupdate + ): + odoo = etree.Element("odoo") + for xml_id in sorted(local_noupdate.keys()): + local_record = local_noupdate[xml_id] + remote_record = None + if xml_id in remote_update and xml_id not in remote_noupdate: + remote_record = remote_update[xml_id] + elif xml_id in remote_noupdate: + remote_record = remote_noupdate[xml_id] + + if "." in xml_id: + module_xmlid = xml_id.split(".", 1)[0] + else: + module_xmlid = "" + + if remote_record is None and not module_xmlid: + continue + + element = etree.Element( + "record", id=xml_id, model=local_record.attrib["model"] + ) + # Add forcecreate attribute if exists + if local_record.attrib.get("forcecreate"): + element.attrib["forcecreate"] = local_record.attrib["forcecreate"] + record_remote_dict = self._get_node_dict(remote_record) + record_local_dict = self._get_node_dict(local_record) + for key in sorted(record_remote_dict.keys()): + if not local_record.xpath(key): + # The element is no longer present. + # Does the field still exist? + if record_remote_dict[key].tag == "field": + field_name = remote_record.xpath(key)[0].attrib.get("name") + if ( + field_name + not in self.env[local_record.attrib["model"]]._fields.keys() + ): + continue + # Overwrite an existing value with an empty one. + attribs = deepcopy(record_remote_dict[key]).attrib + for attr in ["eval", "ref"]: + if attr in attribs: + del attribs[attr] + element.append(etree.Element(record_remote_dict[key].tag, attribs)) + else: + oldrepr = self._get_node_value(record_remote_dict[key]) + newrepr = self._get_node_value(record_local_dict[key]) + + if oldrepr != newrepr: + element.append(deepcopy(record_local_dict[key])) + + for key in sorted(record_local_dict.keys()): + if remote_record is None or not remote_record.xpath(key): + element.append(deepcopy(record_local_dict[key])) + + if len(element): + odoo.append(element) + + if not len(odoo): + return "" + return etree.tostring( + etree.ElementTree(odoo), + pretty_print=True, + xml_declaration=True, + encoding="utf-8", + ).decode("utf-8") + + @staticmethod + def _update_node(target, source): + for element in source: + if "name" in element.attrib: + query = "./{}[@name='{}']".format(element.tag, element.attrib["name"]) + else: + # query = "./%s" % element.tag + continue + for existing in target.xpath(query): + target.remove(existing) + target.append(element) + + @classmethod + def _process_data_node( + self, data_node, records_update, records_noupdate, module_name + ): + noupdate = nodeattr2bool(data_node, "noupdate", False) + for record in data_node.xpath("./record"): + self._process_record_node( + record, noupdate, records_update, records_noupdate, module_name + ) + + @classmethod + def _process_record_node( + self, record, noupdate, records_update, records_noupdate, module_name + ): + xml_id = record.get("id") + if not xml_id: + return + if "." in xml_id and xml_id.startswith(module_name + "."): + xml_id = xml_id[len(module_name) + 1 :] + for records in records_noupdate, records_update: + # records can occur multiple times in the same module + # with different noupdate settings + if xml_id in records: + # merge records (overwriting an existing element + # with the same tag). The order processing the + # various directives from the manifest is + # important here + self._update_node(records[xml_id], record) + break + else: + target_dict = records_noupdate if noupdate else records_update + target_dict[xml_id] = record + + @classmethod + def _parse_files(self, xml_files, module_name): + records_update = {} + records_noupdate = {} + parser = etree.XMLParser( + remove_blank_text=True, + strip_cdata=False, + ) + for xml_file in xml_files: + try: + # This is for a final correct pretty print + # Ref.: https://stackoverflow.com/a/7904066 + # Also don't strip CDATA tags as needed for HTML content + root_node = etree.fromstring(xml_file.encode("utf-8"), parser=parser) + except etree.XMLSyntaxError: + continue + # Support xml files with root Element either odoo or openerp + # Condition: each xml file should have only one root element + # {, or —rarely— }; + root_node_noupdate = nodeattr2bool(root_node, "noupdate", False) + if root_node.tag not in ("openerp", "odoo", "data"): + raise ValidationError( + _( + "Unexpected root Element: %s in file: %s" + % (tree.getroot(), xml_file) + ) + ) + for node in root_node: + if node.tag == "data": + self._process_data_node( + node, records_update, records_noupdate, module_name + ) + elif node.tag == "record": + self._process_record_node( + node, + root_node_noupdate, + records_update, + records_noupdate, + module_name, + ) + + return records_update, records_noupdate + + def generate_noupdate_changes(self): + """Communicate with the remote server to fetch all xml data records + per module, and generate a diff in XML format that can be imported + from the module's migration script using openupgrade.load_data() + """ + self.ensure_one() + connection = self.config_id.get_connection() + remote_record_obj = self._get_remote_model(connection, "record") + local_record_obj = self.env["upgrade.record"] + local_modules = local_record_obj.list_modules() + for remote_module in remote_record_obj.list_modules(): + local_module = renamed_modules.get(remote_module, remote_module) + if local_module not in local_modules: + continue + remote_files = remote_record_obj.get_xml_records(remote_module) + local_files = local_record_obj.get_xml_records(local_module) + remote_update, remote_noupdate = self._parse_files( + remote_files, remote_module + ) + local_update, local_noupdate = self._parse_files(local_files, local_module) + diff = self._get_xml_diff( + remote_update, remote_noupdate, local_update, local_noupdate + ) + if diff: + module = self.env["ir.module.module"].search( + [("name", "=", local_module)] + ) + self._write_file( + local_module, + module.installed_version, + diff, + filename="noupdate_changes.xml", + ) + return True diff --git a/upgrade_analysis/models/upgrade_record.py b/upgrade_analysis/models/upgrade_record.py index 89426c308ce..6aa07cab63c 100644 --- a/upgrade_analysis/models/upgrade_record.py +++ b/upgrade_analysis/models/upgrade_record.py @@ -1,8 +1,16 @@ # Copyright 2011-2015 Therp BV -# Copyright 2016 Opener B.V. +# Copyright 2016-2020 Opener B.V. +# Copyright 2019 Eficent +# Copyright 2020 GRAP # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +import ast +import os + from odoo import api, fields, models +from odoo.exceptions import ValidationError +from odoo.modules.module import MANIFEST_NAMES, get_module_path +from odoo.tools.translate import _ class UpgradeRecord(models.Model): @@ -112,3 +120,41 @@ def field_dump(self): repre.update({x.name: x.value for x in record.attribute_ids}) data.append(repre) return data + + @api.model + def list_modules(self): + """ Return the set of covered modules """ + self.env.cr.execute( + """SELECT DISTINCT(module) FROM upgrade_record + ORDER BY module""" + ) + return [module for module, in self.env.cr.fetchall()] + + @staticmethod + def _read_manifest(addon_dir): + for manifest_name in MANIFEST_NAMES: + if os.access(os.path.join(addon_dir, manifest_name), os.R_OK): + with open(os.path.join(addon_dir, manifest_name), "r") as f: + manifest_string = f.read() + return ast.literal_eval(manifest_string) + raise ValidationError(_("No manifest found in %s" % addon_dir)) + + @api.model + def get_xml_records(self, module): + """ Return all XML records from the given module """ + addon_dir = get_module_path(module) + manifest = self._read_manifest(addon_dir) + # The order of the keys are important. + # Load files in the same order as in + # module/loading.py:load_module_graph + files = [] + for key in ["init_xml", "update_xml", "data"]: + if not manifest.get(key): + continue + for xml_file in manifest[key]: + if not xml_file.lower().endswith(".xml"): + continue + parts = xml_file.split("/") + with open(os.path.join(addon_dir, *parts), "r") as xml_handle: + files.append(xml_handle.read()) + return files