Kucalc: API copy status, Take 2

src/dev/project/start_hub.py

@@ -17,8 +17,9 @@
 ports = util.get_ports()
 base_url = util.base_url()
 
+kucalc = '--kucalc' if len(sys.argv) > 1 and sys.argv[1] == 'kucalc' else ''
 
-cmd = "cd ../../ && . smc-env &&  service_hub.py --dev --foreground --hostname=0.0.0.0 --port={hub_port} --share_port=0 --proxy_port=0 --gap=0 --mentions --base_url={base_url} {test} start".format(
-    base_url=base_url, hub_port=ports['hub'], test=util.test())
+cmd = "cd ../../ && . smc-env &&  service_hub.py --dev --foreground --hostname=0.0.0.0 --port={hub_port} --share_port=0 --proxy_port=0 --gap=0 --mentions --base_url={base_url} {test} {kucalc} start".format(
+    base_url=base_url, hub_port=ports['hub'], test=util.test(), kucalc=kucalc)
 print(cmd)
 util.cmd(cmd)

src/smc-hub/client.coffee

@@ -1095,6 +1095,9 @@ class exports.Client extends EventEmitter
             @error_to_client(id:mesg.id, error:"src_path must be defined")
             return
 
+        locals =
+            copy_id : undefined
+
         async.series([
             (cb) =>
                 # Check permissions for the source and target projects (in parallel) --
@@ -1145,12 +1148,25 @@ class exports.Client extends EventEmitter
                                 backup            : mesg.backup
                                 timeout           : mesg.timeout
                                 exclude_history   : mesg.exclude_history
-                                cb                : cb
+                                wait_until_done   : mesg.wait_until_done
+                                cb                : (err, copy_id) =>
+                                    if err
+                                        cb(err)
+                                    else
+                                        locals.copy_id = copy_id
+                                        cb()
         ], (err) =>
             if err
                 @error_to_client(id:mesg.id, error:err)
             else
-                @push_to_client(message.success(id:mesg.id))
+                # we only expect a copy_id in kucalc mode
+                if locals.copy_id?
+                    resp = message.copy_path_between_projects_response
+                                                        id           : mesg.id
+                                                        copy_path_id : locals.copy_id
+                    @push_to_client(resp)
+                else
+                    @push_to_client(message.success(id:mesg.id))
         )
 
     mesg_copy_path_status: (mesg) =>

src/smc-hub/compute-client.coffee

@@ -1881,6 +1881,7 @@ class ProjectClient extends EventEmitter
             exclude_history   : false
             timeout           : 5*60
             bwlimit           : undefined
+            wait_until_done   : undefined # not used, only relevant for the kucalc variant
             cb                : required
         dbg = @dbg("copy_path(#{opts.path} to #{opts.target_project_id})")
         dbg("copy a path using rsync from one project to another")

src/smc-hub/kucalc/compute-client.coffee

@@ -389,6 +389,7 @@ class Project extends EventEmitter
             exclude_history   : undefined
             timeout           : undefined
             bwlimit           : undefined
+            wait_until_done   : 'true'  # by default, wait until done. false only gives the ID to query the status later
             cb                : undefined
         if not opts.target_project_id
             opts.target_project_id = @project_id
@@ -430,20 +431,25 @@ class Project extends EventEmitter
                     # but just in case, logically we have to check this case.
                     cb()
                     return
-                dbg('waiting for copy to finish...')
-                handle_change = =>
-                    obj = synctable.get(copy_id)
-                    if obj?.get('started')
-                        dbg("copy started...")
-                    if obj?.get('finished')
-                        dbg("copy finished!")
-                        synctable.removeListener('change', handle_change)
-                        cb(obj.get('error'))
-                synctable.on('change', handle_change)
+                if opts.wait_until_done == 'true' or opts.wait_until_done == true
+                    dbg('waiting for copy to finish...')
+                    handle_change = =>
+                        @active()
+                        obj = synctable.get(copy_id)
+                        if obj?.get('started')
+                            dbg("copy started...")
+                        if obj?.get('finished')
+                            dbg("copy finished!")
+                            synctable.removeListener('change', handle_change)
+                            cb(obj.get('error'))
+                    synctable.on('change', handle_change)
+                else
+                    dbg('NOT waiting for copy to finish...')
+                    cb()
         ], (err) =>
             @active()
             dbg('done', err)
-            opts.cb?(err)
+            opts.cb?(err, copy_id)
         )
 
     directory_listing: (opts) =>

src/smc-hub/test/api/copy-api.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# silly helper script to remind hsy how to query the DB from withing a cc-in-cc project
+
+set -evx
+
+PORT=${PORT:-56754}
+CCinCC=${CCinCC:-"14eed217-2d3c-4975-a381-b69edcb40e0e"}
+
+API=http://localhost:$PORT/$CCinCC/port/$PORT/api/v1
+Q() {
+    curl -X POST -u $KEY: $@
+}
+
+KEY=sk_XB003g8HHcyT1y4S3T5w9IW2
+
+## issue copy, needs hub in "kucalc" mode (if this works at all, though)
+SRC=bc6f81b3-25ad-4d58-ae4a-65649fae4fa5
+TAR=e24ba30d-edcd-479f-8a26-bbe81f38296c
+PTH=x.md
+Q -d src_project_id=$SRC -d src_path=$PTH -d target_project_id=$TAR -d wait_until_done=false $API/copy_path_between_projects
+
+
+## status
+## TODO: use jq to extract the copy_path_id from above, query, wait a few secs, and then query it again
+PATHID=002cdde3-e79c-4d46-a759-f6464a5360c9
+Q -d copy_path_id=$PATHID $API/copy_path_status
+
+echo "double check DB"
+psql -x -c "SELECT * FROM copy_paths WHERE id = '$PATHID';"

src/smc-util/message.js

@@ -1738,6 +1738,11 @@ API(
       exclude_history: {
         init: false,
         desc: "if true, exclude all files of the form `*.sage-history`"
+      },
+      wait_until_done: {
+        init: true,
+        desc:
+          "if false, the operation returns immediately with the copy_path_id for querying copy_path_status"
       }
     },
     desc: `\
@@ -1769,6 +1774,13 @@ Folder \`A\` will be created in target project if it does not exist already.
   })
 );
 
+message({
+  event: "copy_path_between_projects_response",
+  id: required,
+  copy_path_id: undefined,
+  note: "Query copy_path_status with the copy_path_id to learn if the copy operation was successful."
+});
+
 API(
   message2({
     event: "copy_path_status",