Added Schema Diff tool to compare two schemas and generate the differ…

…ence script. 

Currently supported objects are Table, View, Materialized View, Function and Procedure.

Backend comparison of two schemas implemented by: Akshay Joshi

Fixes #3452.

docs/en_US/release_notes_4_18.rst

@@ -9,6 +9,7 @@ This release contains a number of bug fixes and new features since the release o
 New features
 ************
 
+| `Issue #3452 <https://redmine.postgresql.org/issues/3452>`_ -  Added Schema Diff tool to compare two schemas and generate the difference script.
 
 Housekeeping
 ************

web/pgadmin/browser/server_groups/servers/__init__.py

@@ -28,6 +28,7 @@
 from pgadmin.utils.driver import get_driver
 from pgadmin.utils.master_password import get_crypt_key
 from pgadmin.utils.exception import CryptKeyMissing
+from pgadmin.tools.schema_diff.node_registry import SchemaDiffRegistry
 from psycopg2 import Error as psycopg2_Error, OperationalError
 
 
@@ -1627,4 +1628,5 @@ def clear_sshtunnel_password(self, gid, sid):
         )
 
 
+SchemaDiffRegistry(blueprint.node_type, ServerNode)
 ServerNode.register_node_view(blueprint)

web/pgadmin/browser/server_groups/servers/databases/__init__.py

@@ -30,6 +30,8 @@
     make_response as ajax_response, internal_server_error, unauthorized
 from pgadmin.utils.driver import get_driver
 from pgadmin.tools.sqleditor.utils.query_history import QueryHistory
+
+from pgadmin.tools.schema_diff.node_registry import SchemaDiffRegistry
 from pgadmin.model import Server
 
 
@@ -1111,4 +1113,5 @@ def dependencies(self, gid, sid, did):
         )
 
 
+SchemaDiffRegistry(blueprint.node_type, DatabaseView)
 DatabaseView.register_node_view(blueprint)

web/pgadmin/browser/server_groups/servers/databases/schemas/__init__.py

@@ -23,6 +23,7 @@
 from pgadmin.utils.ajax import make_json_response, internal_server_error, \
     make_response as ajax_response, gone, bad_request
 from pgadmin.utils.driver import get_driver
+from pgadmin.tools.schema_diff.node_registry import SchemaDiffRegistry
 
 """
     This module is responsible for generating two nodes
@@ -1023,5 +1024,6 @@ def sql(self, gid, sid, did, scid):
         return ajax_response(response=SQL.strip("\n"))
 
 
+SchemaDiffRegistry(schema_blueprint.node_type, SchemaView)
 SchemaView.register_node_view(schema_blueprint)
 CatalogView.register_node_view(catalog_blueprint)

web/pgadmin/browser/server_groups/servers/databases/schemas/collations/__init__.py

@@ -25,6 +25,8 @@
     make_response as ajax_response, gone
 from pgadmin.utils.compile_template_name import compile_template_path
 from pgadmin.utils.driver import get_driver
+from pgadmin.tools.schema_diff.node_registry import SchemaDiffRegistry
+from pgadmin.tools.schema_diff.compare import SchemaDiffObjectCompare
 
 # If we are in Python3
 if not IS_PY2:
@@ -92,7 +94,7 @@ def node_inode(self):
 blueprint = CollationModule(__name__)
 
 
-class CollationView(PGChildNodeView):
+class CollationView(PGChildNodeView, SchemaDiffObjectCompare):
     """
     This class is responsible for generating routes for Collation node
 
@@ -144,6 +146,10 @@ class CollationView(PGChildNodeView):
     * dependent(gid, sid, did, scid):
       - This function will generate dependent list to show it in dependent
         pane for the selected Collation node.
+
+    * compare(**kwargs):
+      - This function will compare the collation nodes from two different
+        schemas.
     """
 
     node_type = blueprint.node_type
@@ -172,7 +178,8 @@ class CollationView(PGChildNodeView):
         'dependency': [{'get': 'dependencies'}],
         'dependent': [{'get': 'dependents'}],
         'get_collations': [{'get': 'get_collation'},
-                           {'get': 'get_collation'}]
+                           {'get': 'get_collation'}],
+        'compare': [{'get': 'compare'}, {'get': 'compare'}]
     })
 
     def check_precondition(f):
@@ -318,23 +325,36 @@ def properties(self, gid, sid, did, scid, coid):
             JSON of selected collation node
         """
 
+        status, res = self._fetch_properties(scid, coid)
+        if not status:
+            return res
+
+        return ajax_response(
+            response=res,
+            status=200
+        )
+
+    def _fetch_properties(self, scid, coid):
+        """
+        This function fetch the properties for the specified object.
+
+        :param scid: Schema ID
+        :param coid: Collation ID
+        """
+
         SQL = render_template("/".join([self.template_path,
                                         'properties.sql']),
                               scid=scid, coid=coid)
         status, res = self.conn.execute_dict(SQL)
 
         if not status:
-            return internal_server_error(errormsg=res)
+            return False, internal_server_error(errormsg=res)
 
         if len(res['rows']) == 0:
-            return gone(
-                gettext("Could not find the collation object in the database.")
-            )
+            return False, gone(gettext("Could not find the collation "
+                                       "object in the database."))
 
-        return ajax_response(
-            response=res['rows'][0],
-            status=200
-        )
+        return True, res['rows'][0]
 
     @check_precondition
     def get_collation(self, gid, sid, did, scid, coid=None):
@@ -748,5 +768,30 @@ def dependencies(self, gid, sid, did, scid, coid):
             status=200
         )
 
+    @check_precondition
+    def fetch_objects_to_compare(self, sid, did, scid):
+        """
+        This function will fetch the list of all the collations for
+        specified schema id.
+
+        :param sid: Server Id
+        :param did: Database Id
+        :param scid: Schema Id
+        :return:
+        """
+        res = dict()
+        SQL = render_template("/".join([self.template_path,
+                                        'nodes.sql']), scid=scid)
+        status, rset = self.conn.execute_2darray(SQL)
+        if not status:
+            return internal_server_error(errormsg=res)
+
+        for row in rset['rows']:
+            status, data = self._fetch_properties(scid, row['oid'])
+            if status:
+                res[row['name']] = data
+
+        return res
+
 
 CollationView.register_node_view(blueprint)

web/pgadmin/browser/server_groups/servers/databases/schemas/domains/__init__.py

@@ -27,6 +27,8 @@
     make_response as ajax_response, gone
 from pgadmin.utils.compile_template_name import compile_template_path
 from pgadmin.utils.driver import get_driver
+from pgadmin.tools.schema_diff.node_registry import SchemaDiffRegistry
+from pgadmin.tools.schema_diff.compare import SchemaDiffObjectCompare
 
 # If we are in Python3
 if not IS_PY2:
@@ -79,7 +81,7 @@ def script_load(self):
 blueprint = DomainModule(__name__)
 
 
-class DomainView(PGChildNodeView, DataTypeReader):
+class DomainView(PGChildNodeView, DataTypeReader, SchemaDiffObjectCompare):
     """
     class DomainView
 
@@ -138,6 +140,10 @@ class DomainView
 
     * types(gid, sid, did, scid, fnid=None):
       - Returns Data Types.
+
+    * compare(**kwargs):
+      - This function will compare the domain nodes from two different
+        schemas.
     """
 
     node_type = blueprint.node_type
@@ -169,7 +175,8 @@ class DomainView
         'get_collations': [
             {'get': 'get_collations'},
             {'get': 'get_collations'}
-        ]
+        ],
+        'compare': [{'get': 'compare'}, {'get': 'compare'}]
     })
 
     def validate_request(f):
@@ -369,15 +376,31 @@ def properties(self, gid, sid, did, scid, doid):
             scid: Schema Id
             doid: Domain Id
         """
+        status, res = self._fetch_properties(did, scid, doid)
+        if not status:
+            return res
+
+        return ajax_response(
+            response=res,
+            status=200
+        )
 
+    def _fetch_properties(self, did, scid, doid):
+        """
+        This function is used to fecth the properties of specified object.
+        :param did:
+        :param scid:
+        :param doid:
+        :return:
+        """
         SQL = render_template("/".join([self.template_path, 'properties.sql']),
                               scid=scid, doid=doid)
         status, res = self.conn.execute_dict(SQL)
         if not status:
-            return internal_server_error(errormsg=res)
+            return False, internal_server_error(errormsg=res)
 
         if len(res['rows']) == 0:
-            return gone(gettext("""
+            return False, gone(gettext("""
 Could not find the domain in the database.
 It may have been removed by another user or moved to another schema.
 """))
@@ -393,7 +416,7 @@ def properties(self, gid, sid, did, scid, doid):
                               doid=doid)
         status, res = self.conn.execute_dict(SQL)
         if not status:
-            return internal_server_error(errormsg=res)
+            return False, internal_server_error(errormsg=res)
 
         data['constraints'] = res['rows']
 
@@ -406,10 +429,7 @@ def properties(self, gid, sid, did, scid, doid):
         if doid <= self.manager.db_info[did]['datlastsysoid']:
             data['sysdomain'] = True
 
-        return ajax_response(
-            response=data,
-            status=200
-        )
+        return True, data
 
     def _parse_type(self, basetype):
         """
@@ -664,7 +684,7 @@ def update(self, gid, sid, did, scid, doid):
         )
 
     @check_precondition
-    def sql(self, gid, sid, did, scid, doid=None):
+    def sql(self, gid, sid, did, scid, doid=None, return_ajax_response=True):
         """
         Returns the SQL for the Domain object.
 
@@ -674,6 +694,7 @@ def sql(self, gid, sid, did, scid, doid=None):
             did: Database Id
             scid: Schema Id
             doid: Domain Id
+            return_ajax_response:
         """
 
         SQL = render_template("/".join([self.template_path,
@@ -716,6 +737,9 @@ def sql(self, gid, sid, did, scid, doid=None):
 """.format(self.qtIdent(self.conn, data['basensp'], data['name']))
         SQL = sql_header + SQL
 
+        if not return_ajax_response:
+            return SQL.strip('\n')
+
         return ajax_response(response=SQL.strip('\n'))
 
     @check_precondition
@@ -846,5 +870,40 @@ def dependencies(self, gid, sid, did, scid, doid):
             status=200
         )
 
+    @check_precondition
+    def fetch_objects_to_compare(self, sid, did, scid):
+        """
+        This function will fetch the list of all the domains for
+        specified schema id.
+
+        :param sid: Server Id
+        :param did: Database Id
+        :param scid: Schema Id
+        :return:
+        """
+        res = dict()
+        SQL = render_template("/".join([self.template_path,
+                                        'node.sql']), scid=scid)
+        status, rset = self.conn.execute_2darray(SQL)
+        if not status:
+            return internal_server_error(errormsg=res)
+
+        for row in rset['rows']:
+            status, data = self._fetch_properties(did, scid, row['oid'])
+
+            if status:
+                if 'constraints' in data and len(data['constraints']) > 0:
+                    for item in data['constraints']:
+                        # Remove keys that should not be the part
+                        # of comparision.
+                        if 'conoid' in item:
+                            item.pop('conoid')
+                        if 'nspname' in item:
+                            item.pop('nspname')
+
+                res[row['name']] = data
+
+        return res
+
 
 DomainView.register_node_view(blueprint)

web/pgadmin/browser/server_groups/servers/databases/schemas/domains/templates/domains/sql/9.2_plus/get_constraints.sql

@@ -8,7 +8,7 @@ JOIN
 JOIN
     pg_namespace nl ON nl.oid=typnamespace
 LEFT OUTER JOIN
-    pg_description des ON (des.objoid=t.oid AND des.classoid='pg_constraint'::regclass)
+    pg_description des ON (des.objoid=c.oid AND des.classoid='pg_constraint'::regclass)
 WHERE
     contype = 'c' AND contypid =  {{doid}}::oid
 ORDER BY

web/pgadmin/browser/server_groups/servers/databases/schemas/domains/templates/domains/sql/default/get_constraints.sql

@@ -8,7 +8,7 @@ JOIN
 JOIN
     pg_namespace nl ON nl.oid=typnamespace
 LEFT OUTER JOIN
-    pg_description des ON (des.objoid=t.oid AND des.classoid='pg_constraint'::regclass)
+    pg_description des ON (des.objoid=c.oid AND des.classoid='pg_constraint'::regclass)
 WHERE
     contype = 'c'
     AND contypid =  {{doid}}::oid