From ad145c42e1e8bb92a61533540744154729ba1977 Mon Sep 17 00:00:00 2001
From: Ilya Grishnov <ilya.grishnov@tarantool.org>
Date: Tue, 15 Nov 2022 18:01:29 +0300
Subject: [PATCH] api: native crud module support

Adds native api support for crud module [1]
to use it from a connection object.

1. github.com/tarantool/crud

Closes #205
---
 tarantool/connection.py         |   2 +
 tarantool/crud.py               | 748 ++++++++++++++++++++++++++++++++
 test/suites/crud_server_cfg.lua |  64 +++
 3 files changed, 814 insertions(+)
 create mode 100644 tarantool/crud.py
 create mode 100644 test/suites/crud_server_cfg.lua

diff --git a/tarantool/connection.py b/tarantool/connection.py
index 5b4fe608..eb80286d 100644
--- a/tarantool/connection.py
+++ b/tarantool/connection.py
@@ -86,6 +86,7 @@
     warn
 )
 from tarantool.schema import Schema
+from tarantool.crud import Crud
 from tarantool.utils import (
     greeting_decode,
     version_id,
@@ -516,6 +517,7 @@ def __init__(self, host, port,
             IPROTO_FEATURE_WATCHERS: False,
             IPROTO_FEATURE_GRACEFUL_SHUTDOWN: False,
         }
+        self.crud = Crud(self)
 
         if connect_now:
             self.connect()
diff --git a/tarantool/crud.py b/tarantool/crud.py
new file mode 100644
index 00000000..f94441e9
--- /dev/null
+++ b/tarantool/crud.py
@@ -0,0 +1,748 @@
+class Response(object):
+    def __init__(self, response):
+        """
+        TODO: write docs
+        """
+
+        if isinstance(response, dict):
+            for response_field_name in response.keys():
+                setattr(self, response_field_name, response[response_field_name])
+        elif isinstance(response, str):
+            self.str = response
+            self.err = response
+
+
+class Result(Response):
+    """
+    Contains result's fields from result variable of crud module operation.
+    """
+
+
+class Error(Response):
+    """
+    Contains error's fields from error variable of crud module operation.
+    """
+
+
+class Crud(object):
+    """
+    Contains crud module methods.
+    """
+
+    def __init__(self, con):
+        """
+        TODO: write docs
+        """
+
+        self.con = con
+        self._crud_module_init_complete = False
+
+    def _crud_module_init(self):
+        self.require_ok = self.con.eval("ok, crud = pcall(require, 'crud') return ok").data[0]
+        self.con.eval("""
+            function crud_pairs(space_name, conditions, opts) 
+                res = {} 
+                for _, v in crud.pairs(space_name, conditions, opts) do 
+                    table.insert(res, v) 
+                end 
+                return res 
+            end
+        """)
+        self.con.eval("""
+            function pcall_crud_pairs(space_name, conditions, opts) 
+                ok, res = pcall(crud_pairs, space_name, conditions, opts) 
+                return ok, res 
+            end
+        """)
+        self._crud_module_init_complete = True
+
+    def insert(self, space_name, values, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(values, (tuple, list))
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.insert", space_name, values, opts)
+
+        res = None
+        if crud_resp[0] is not None:
+            res = Result(crud_resp[0])
+
+        err = None
+        if crud_resp[1] is not None:
+            err = Error(crud_resp[1])
+
+        return res, err
+
+    def insert_object(self, space_name, values, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(values, dict)
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.insert_object", space_name, values, opts)
+        
+        res = None
+        if crud_resp[0] is not None:
+            res = Result(crud_resp[0])
+
+        err = None
+        if crud_resp[1] is not None:
+            err = Error(crud_resp[1])
+
+        return res, err
+
+    def insert_many(self, space_name, values, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(values, (tuple, list))
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.insert_many", space_name, values, opts)
+
+        res = None
+        if crud_resp[0] is not None:
+            res = Result(crud_resp[0])
+
+        errs = list()
+        if crud_resp[1] is not None:
+            for err in crud_resp[1]:
+                errs.append(Error(err))
+
+        return res, errs
+
+    def insert_object_many(self, space_name, values, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(values, (tuple, list))
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.insert_object_many", space_name, values, opts)
+
+        res = None
+        if crud_resp[0] is not None:
+            res = Result(crud_resp[0])
+
+        errs = list()
+        if crud_resp[1] is not None:
+            for err in crud_resp[1]:
+                errs.append(Error(err))
+
+        return res, errs
+
+    def get(self, space_name, key, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.get", space_name, key, opts)
+
+        res = None
+        if crud_resp[0] is not None:
+            res = Result(crud_resp[0])
+
+        err = None
+        if crud_resp[1] is not None:
+            err = Error(crud_resp[1])
+
+        return res, err
+
+    def update(self, space_name, key, operations=[], opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(operations, list)
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.update", space_name, key, operations, opts)
+
+        res = None
+        if crud_resp[0] is not None:
+            res = Result(crud_resp[0])
+
+        err = None
+        if crud_resp[1] is not None:
+            err = Error(crud_resp[1])
+
+        return res, err
+
+    def delete(self, space_name, key, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.delete", space_name, key, opts)
+
+        res = None
+        if crud_resp[0] is not None:
+            res = Result(crud_resp[0])
+
+        err = None
+        if crud_resp[1] is not None:
+            err = Error(crud_resp[1])
+
+        return res, err
+
+    def replace(self, space_name, values, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(values, (tuple, list))
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.replace", space_name, values, opts)
+
+        res = None
+        if crud_resp[0] is not None:
+            res = Result(crud_resp[0])
+
+        err = None
+        if crud_resp[1] is not None:
+            err = Error(crud_resp[1])
+
+        return res, err
+
+
+    def replace_object(self, space_name, values, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(values, dict)
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.replace_object", space_name, values, opts)
+        
+        res = None
+        if crud_resp[0] is not None:
+            res = Result(crud_resp[0])
+
+        err = None
+        if crud_resp[1] is not None:
+            err = Error(crud_resp[1])
+
+        return res, err
+
+    def replace_many(self, space_name, values, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(values, (tuple, list))
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.replace_many", space_name, values, opts)
+
+        res = None
+        if crud_resp[0] is not None:
+            res = Result(crud_resp[0])
+
+        errs = list()
+        if crud_resp[1] is not None:
+            for err in crud_resp[1]:
+                errs.append(Error(err))
+
+        return res, errs
+
+    def replace_object_many(self, space_name, values, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(values, (tuple, list))
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.replace_object_many", space_name, values, opts)
+
+        res = None
+        if crud_resp[0] is not None:
+            res = Result(crud_resp[0])
+
+        errs = list()
+        if crud_resp[1] is not None:
+            for err in crud_resp[1]:
+                errs.append(Error(err))
+
+        return res, errs
+
+    def upsert(self, space_name, values, operations=[], opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(values, (tuple, list))
+        assert isinstance(operations, list)
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.upsert", space_name, values, operations, opts)
+
+        res = None
+        if crud_resp[0] is not None:
+            res = Result(crud_resp[0])
+
+        err = None
+        if crud_resp[1] is not None:
+            err = Error(crud_resp[1])
+
+        return res, err
+
+
+    def upsert_object(self, space_name, values, operations=[], opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(values, dict)
+        assert isinstance(operations, list)
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.upsert_object", space_name, values, operations, opts)
+        
+        res = None
+        if crud_resp[0] is not None:
+            res = Result(crud_resp[0])
+
+        err = None
+        if crud_resp[1] is not None:
+            err = Error(crud_resp[1])
+
+        return res, err
+
+    def upsert_many(self, space_name, values_operation, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(values_operation, (tuple, list))
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.upsert_many", space_name, values_operation, opts)
+
+        res = None
+        if crud_resp[0] is not None:
+            res = Result(crud_resp[0])
+
+        errs = list()
+        if crud_resp[1] is not None:
+            for err in crud_resp[1]:
+                errs.append(Error(err))
+
+        return res, errs
+
+    def upsert_object_many(self, space_name, values_operation, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(values_operation, (tuple, list))
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.upsert_object_many", space_name, values_operation, opts)
+
+        res = None
+        if crud_resp[0] is not None:
+            res = Result(crud_resp[0])
+
+        errs = list()
+        if crud_resp[1] is not None:
+            for err in crud_resp[1]:
+                errs.append(Error(err))
+
+        return res, errs
+
+    def select(self, space_name, conditions={}, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(conditions, dict)
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.select", space_name, conditions, opts)
+
+        res = None
+        if crud_resp[0] is not None:
+            res = Result(crud_resp[0])
+
+        err = None
+        if crud_resp[1] is not None:
+            err = Error(crud_resp[1])
+
+        return res, err
+
+    def pairs(self, space_name, conditions={}, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(conditions, dict)
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("pcall_crud_pairs", space_name, conditions, opts)
+
+        if not crud_resp[0]:
+            raise Exception(crud_resp[1])
+
+        return crud_resp[1]
+
+    def min(self, space_name, index_name, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.min", space_name, index_name, opts)
+
+        res = None
+        if crud_resp[0] is not None:
+            res = Result(crud_resp[0])
+
+        err = None
+        if crud_resp[1] is not None:
+            err = Error(crud_resp[1])
+
+        return res, err
+
+    def max(self, space_name, index_name, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.max", space_name, index_name, opts)
+
+        res = None
+        if crud_resp[0] is not None:
+            res = Result(crud_resp[0])
+
+        err = None
+        if crud_resp[1] is not None:
+            err = Error(crud_resp[1])
+
+        return res, err
+
+    def cut_rows(self, rows, metadata, fields):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(rows, (tuple, list))
+        assert isinstance(metadata, (tuple, list))
+        assert isinstance(fields, (tuple, list))
+
+        crud_resp = self.con.call("crud.cut_rows", rows, metadata, fields)
+
+        # NOTE: in absence of an error, crud does not give variable err as nil (as in most cases)
+        res, err = None, None
+        if len(crud_resp) == 1:
+            res = Result(crud_resp[0])
+        else:
+            err = Error(crud_resp[1])
+
+        return res, err
+
+    def cut_objects(self, objects, fields):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(objects, (tuple, list))
+        assert isinstance(fields, (tuple, list))
+
+        crud_resp = self.con.call("crud.cut_objects", objects, fields)
+
+        return crud_resp
+
+    def truncate(self, space_name, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.truncate", space_name, opts)
+
+        # NOTE: in absence of an error, crud does not give variable err as nil (as in most cases)
+        res, err = None, None
+        if len(crud_resp) == 1:
+            res = crud_resp[0]
+        else:
+            err = Error(crud_resp[1])
+
+        return res, err
+    
+    def len(self, space_name, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.len", space_name, opts)
+
+        # NOTE: in absence of an error, crud does not give variable err as nil (as in most cases)
+        res, err = None, None
+        if len(crud_resp) == 1:
+            res = crud_resp[0]
+        else:
+            err = Error(crud_resp[1])
+
+        return res, err
+
+    def storage_info(self, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.storage_info", opts)
+
+        # NOTE: in absence of an error, crud does not give variable err as nil (as in most cases)
+        res, err = None, None
+        if len(crud_resp) == 1:
+            res = crud_resp[0]
+        else:
+            err = Error(crud_resp[1])
+
+        return res, err
+
+    def count(self, space_name, conditions={}, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(space_name, str)
+        assert isinstance(conditions, dict)
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.count", space_name, conditions, opts)
+
+        res = None
+        if crud_resp[0] is not None:
+            res = crud_resp[0]
+
+        err = None
+        if crud_resp[1] is not None:
+            err = Error(crud_resp[1])
+
+        return res, err
+
+    def cfg(self, opts={}):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(opts, dict)
+
+        crud_resp = self.con.call("crud.cfg", opts)
+
+        return Result(crud_resp.data[0])
+
+    def stats(self, space_name=None):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        if space_name is not None:
+            assert isinstance(space_name, str)
+
+        crud_resp = self.con.call("crud.stats", space_name)
+
+        res = None
+        if len(crud_resp.data[0]) > 0:
+            res = Result(crud_resp.data[0])
+
+        return res
+
+    def reset_stats(self):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        crud_resp = self.con.call("crud.reset_stats")
+
+        return crud_resp.data[0]
+
+    def init_router(self):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        self.con.call("crud.init_router")
+
+    def init_storage(self):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        self.con.call("crud.init_storage")
+
+    def stop_router(self):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        self.con.call("crud.stop_router")
+
+    def stop_storage(self):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        self.con.call("crud.stop_storage")
+
+    def unflatten_rows(self, rows, metadata):
+        """
+        TODO: write docs
+        """
+
+        if not self._crud_module_init_complete:
+            self._crud_module_init()
+        assert self.require_ok == True, 'Problem with require of crud module on the remote machine'
+        assert isinstance(rows, (tuple, list))
+        assert isinstance(metadata, (tuple, list))
+
+        crud_resp = self.con.call("crud.unflatten_rows", rows, metadata)
+
+        # NOTE: in absence of an error, crud does not give variable err as nil (as in most cases)
+        res, err = None, None
+        if len(crud_resp) == 1:
+            res = crud_resp[0]
+        else:
+            err = Error(crud_resp[1])
+
+        return res, err
diff --git a/test/suites/crud_server_cfg.lua b/test/suites/crud_server_cfg.lua
new file mode 100644
index 00000000..c79a36a1
--- /dev/null
+++ b/test/suites/crud_server_cfg.lua
@@ -0,0 +1,64 @@
+#!/usr/bin/env tarantool
+
+os = require('os')
+crud = require('crud')
+vshard = require('vshard')
+
+box.cfg{
+    listen = 3301,
+}
+
+box.schema.user.grant(
+    'guest',
+    'read,write,execute',
+    'universe'
+)
+box.schema.create_space(
+ 'tester', {
+  format = {
+    {name = 'id', type = 'unsigned'},
+    {name = 'bucket_id', type = 'unsigned'},
+    {name = 'name', type = 'string'},
+  }
+})
+box.space.tester:create_index('primary_index', {
+    parts = {
+        {field = 1, type = 'unsigned'},
+    },
+})
+box.space.tester:create_index('bucket_id', {
+    parts = {
+        {field = 2, type = 'unsigned'},
+    },
+    unique = false,
+})
+
+-- Setup vshard.
+_G.vshard = vshard
+box.once('guest', function()
+    box.schema.user.grant('guest', 'super')
+end)
+local uri = 'guest@localhost:' .. '3301'
+local cfg = {
+    bucket_count = 3000,
+    sharding = {
+        [box.info().cluster.uuid] = {
+            replicas = {
+                [box.info().uuid] = {
+                    uri = uri,
+                    name = 'storage',
+                    master = true,
+                },
+            },
+        },
+    },
+}
+vshard.storage.cfg(cfg, box.info().uuid)
+vshard.router.cfg(cfg)
+vshard.router.bootstrap()
+
+-- Initialize crud.
+crud.init_storage()
+crud.init_router()
+
+require('console').start()