From b2535295af34af54bfbdd832cf448bc79f12dd49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Acun=CC=83a?= Date: Tue, 17 May 2016 11:35:06 -0300 Subject: [PATCH 1/8] Using run_as_user to get permissions to files --- lib/agent/actions/fileretrieval/index.js | 110 ++++----------- .../{files_storage.js => storage.js} | 15 +- lib/agent/actions/fileretrieval/upload.js | 131 ++++++++++++++++++ 3 files changed, 162 insertions(+), 94 deletions(-) rename lib/agent/actions/fileretrieval/{files_storage.js => storage.js} (86%) create mode 100644 lib/agent/actions/fileretrieval/upload.js diff --git a/lib/agent/actions/fileretrieval/index.js b/lib/agent/actions/fileretrieval/index.js index bc7380980..02f27e446 100644 --- a/lib/agent/actions/fileretrieval/index.js +++ b/lib/agent/actions/fileretrieval/index.js @@ -13,107 +13,45 @@ var fs = require('fs'), common = require('./../../common'), needle = require('needle'), join = require('path').join, - api = require('./../../plugins/control-panel/api'), - files = require('./files_storage'), - EventEmitter = require('events').EventEmitter; + files = require('./storage'), + Emitter = require('events').EventEmitter; var system = common.system, - config = common.config, - protocol = config._values['control-panel'].protocol, - host = config._values['control-panel'].host, - url = protocol + '://' + host, + run_as_user = common.system.run_as_user, node_bin = path.join(system.paths.current, 'bin', 'node'), - logger = common.logger, - os_name = process.platform.replace('darwin', 'mac').replace('win32', 'windows'); + logger = common.logger; var em, cp; -var UPLOAD_SERVER = url + '/upload/upload', - RESUMABLE_HEADER = 'X-Prey-Upload-Resumable', - OPEN_TIMEOUT = 180000, - READ_TIMEOUT = 5000; - +// check_pending_files is used to resume any files that might been pending. It's called from +// filesagent/providers/network. exports.check_pending_files = function() { files.run_stored(); } exports.start = function(options, cb) { - var file_path = options.path, - file_id = options.file_id, - file_size = parseInt(options.size), - user = options.user; - - em = em || new EventEmitter(); - if (cb) cb(null, em); - - if (!options.resumable) - files.store(file_id, file_path, file_size, user); - - var file = { - total: 0, - path: file_path, - user: user, - id: file_id, - size: file_size, - resumable: options.resumable - } - - retrieve_file(file, function() { - files.del(file.id); - }); -} - -function retrieve_file(file, cb) { - if (file.resumable) { - var url = UPLOAD_SERVER + '?uploadID=' + file.id; - // Make a call to get the last byte processed by the upload server - // before the disconnection in order to resume the upload from that position - needle.request('get', url, null, function(err, res) { - if (err) return em.emit(err); - var data = JSON.parse(res.body); - file.total = data.Total; - get_file(file, cb); - }) - return; - } - get_file(file, cb); -} - -function get_file(file, cb) { - var buffsize = (file.size == 0) ? 1 : (file.size - file.total); - var buf = new Buffer(buffsize); - var fd = fs.openSync(file.path, "r"); - - fs.read(fd, buf, 0, file.size - file.total, file.total, function(err, read, buf) { - if (err) return em.emit(err); - upload_file(file, buf, cb); - }) -} - -function upload_file(file, buf, cb) { - var options = { - open_timeout: OPEN_TIMEOUT, - read_timeout: READ_TIMEOUT + console.log(options); + var opts = { + user: options.user, + bin: node_bin, + type: 'exec', + args: [path.join(__dirname, 'upload.js'), '"' + options.path + '"', options.user, '"'+options.name+'"', options.size, options.file_id, options.port], + opts: { + env: process.env + } }; - - if (file.total > 0) { - RESUMABLE_HEADER = file.total; - } - var url = UPLOAD_SERVER + '?uploadID='+file.id; - needle.post(url, buf, options, function(err, res) { + em = em || new Emitter(); + + run_as_user(opts, function(err, out) { if (err) { - logger.error(err) - return em.emit(err); + logger.error("Upload error: " + err.message); + return; } - var out = res.statusCode; - if (out !== 200 && out !== 201) { - var err = new Error('There was an error communicating with the server'); - logger.error(err) - em.emit(err); - } - cb(); - }) + logger.info("Ran as user:" + out); + }); + + cb(null, em); em.emit('end'); } diff --git a/lib/agent/actions/fileretrieval/files_storage.js b/lib/agent/actions/fileretrieval/storage.js similarity index 86% rename from lib/agent/actions/fileretrieval/files_storage.js rename to lib/agent/actions/fileretrieval/storage.js index 37a674f41..1f87b7013 100644 --- a/lib/agent/actions/fileretrieval/files_storage.js +++ b/lib/agent/actions/fileretrieval/storage.js @@ -15,21 +15,21 @@ var exist = function(id, cb) { var key = NAMESPACE + id; storage.all(function(err, files) { if (err) - return logger.error(err.message); + return err.message; if (files[key]) return cb(true); return cb(false); - }) + }); } -exports.store = function(id, path, size, user) { +exports.store = function(id, path, size, user, name) { exist(id, function(cb) { if (cb == false) { - logger.debug('Storing file_id in DB: ' + id); var opts = { path: path, size: size, - user: user + user: user, + name: name } storage.set(NAMESPACE + id, opts); } @@ -37,7 +37,6 @@ exports.store = function(id, path, size, user) { } exports.del = function(id) { - logger.debug('Removing file_id from DB: ' + id); storage.del(NAMESPACE + id); } @@ -49,7 +48,6 @@ exports.run_stored = function(cb) { var count = Object.keys(files).length; if (count <= 0) return; - logger.warn('Re-uploading ' + count + ' pending files.'); for (key in files) { @@ -57,10 +55,11 @@ exports.run_stored = function(cb) { continue; } var opts = { - file_id: key.substring(3, key.length), path: files[key].path, user: files[key].user, + name: files[key].name, size: files[key].size, + file_id: key.substring(3, key.length), resumable: true } fileretrieval.start(opts, cb); diff --git a/lib/agent/actions/fileretrieval/upload.js b/lib/agent/actions/fileretrieval/upload.js new file mode 100644 index 000000000..25d2a7609 --- /dev/null +++ b/lib/agent/actions/fileretrieval/upload.js @@ -0,0 +1,131 @@ +#!/usr/bin/env node + +var files = require('./storage'), + fs = require('fs'), + path = require('path'), + mime = require('mime'), + common = require('./../../common'), + needle = require('needle'); + +var config = common.config, + protocol = config._values['control-panel'].protocol, + host = config._values['control-panel'].host, + url = protocol + '://' + host; + +var UPLOAD_SERVER = url + '/upload/upload', + RESUMABLE_HEADER = 'X-Prey-Upload-Resumable', + OPEN_TIMEOUT = 180000, + READ_TIMEOUT = 5000; + +var PATH = 2, + USER = 3, + NAME = 4, + SIZE = 5, + FILE_ID = 6, + PORT = 7; + +function main() { + var argv = process.argv; + var options = { + path: argv[PATH], + user: argv[USER], + name: argv[NAME], + size: argv[SIZE], + file_id: argv[FILE_ID], + port: argv[PORT] + } + Main(options, function(err) { + if (err) { + console.error(err); + return; + } + files.del(options.file_id); + }); +} + +function Main(options, cb) { + var file_path = options.path, + file_id = options.file_id, + file_size = parseInt(options.size), + file_name = options.name, + user = options.user; + + console.log("Uploading file: ", file_path, file_id); + + if (!options.resumable) { + files.store(file_id, file_path, file_size, user, file_name); + } + + var file = { + total: 0, + path: file_path, + user: user, + id: file_id, + size: file_size, + resumable: options.resumable + } + retrieve_file(file, cb); +} + +function retrieve_file(file, cb) { + if (file.resumable) { + console.log("Resumable file:", file.id); + var url = UPLOAD_SERVER + '?uploadID=' + file.id; + // Make a call to get the last byte processed by the upload server + // in order to resume the upload from that position. + needle.request('get', url, null, function(err, res) { + if (err) { + cb(err); + return; + } + var data = JSON.parse(res.body); + file.total = data.Total; + get_file(file, cb); + }) + return; + } + get_file(file, cb); +} + +function get_file(file, cb) { + var buffsize = (file.size == 0) ? 1 : (file.size - file.total); + var buf = new Buffer(buffsize); + var fd = fs.openSync(file.path, "r"); + + fs.read(fd, buf, 0, file.size - file.total, file.total, function(err, read, buf) { + if (err) { + cb(err); + return; + } + upload_file(file, buf, cb); + }) +} + +function upload_file(file, buf, cb) { + var options = { + open_timeout: OPEN_TIMEOUT, + read_timeout: READ_TIMEOUT + }; + + if (file.total > 0) { + RESUMABLE_HEADER = file.total; + } + var url = UPLOAD_SERVER + '?uploadID='+file.id; + + needle.post(url, buf, options, function(err, res) { + if (err) { + cb(err); + return; + } + var out = res.statusCode; + if (out !== 200 && out !== 201) { + var err = new Error('There was an error communicating with the server'); + cb(err); + return; + } + console.log("File succesfuly uploaded:", file.id); + cb(null); // delete files + }) +} + +main(); \ No newline at end of file From 0f37172a22e2cb07f7474250bec4275b47ab4213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Acun=CC=83a?= Date: Tue, 17 May 2016 15:12:57 -0300 Subject: [PATCH 2/8] =?UTF-8?q?Signed-off-by:=20Javier=20Acun=CC=83a=20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/agent/actions/fileretrieval/index.js | 21 ++++++++++----- lib/agent/actions/fileretrieval/upload.js | 32 +++++++++++------------ 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/lib/agent/actions/fileretrieval/index.js b/lib/agent/actions/fileretrieval/index.js index 02f27e446..5178dd4cd 100644 --- a/lib/agent/actions/fileretrieval/index.js +++ b/lib/agent/actions/fileretrieval/index.js @@ -14,12 +14,12 @@ var fs = require('fs'), needle = require('needle'), join = require('path').join, files = require('./storage'), - Emitter = require('events').EventEmitter; + Emitter = require('events').EventEmitter; -var system = common.system, +var system = common.system, run_as_user = common.system.run_as_user, - node_bin = path.join(system.paths.current, 'bin', 'node'), - logger = common.logger; + node_bin = path.join(system.paths.current, 'bin', 'node'), + logger = common.logger; var em, cp; @@ -31,27 +31,34 @@ exports.check_pending_files = function() { } exports.start = function(options, cb) { - console.log(options); + if (!options.resumable) { + options.resumable = false; + } var opts = { user: options.user, bin: node_bin, type: 'exec', - args: [path.join(__dirname, 'upload.js'), '"' + options.path + '"', options.user, '"'+options.name+'"', options.size, options.file_id, options.port], + args: [path.join(__dirname, 'upload.js'), '"' + options.path + '"', options.user, '"'+options.name+'"', options.size, options.file_id, options.resumable, options.port], opts: { env: process.env } }; em = em || new Emitter(); + files.store(options.file_id, options.path, options.size, options.user, options.name); + run_as_user(opts, function(err, out) { if (err) { logger.error("Upload error: " + err.message); return; } logger.info("Ran as user:" + out); + + files.del(options.file_id); + }); - cb(null, em); + if (cb) cb(null, em); em.emit('end'); } diff --git a/lib/agent/actions/fileretrieval/upload.js b/lib/agent/actions/fileretrieval/upload.js index 25d2a7609..7dc0e3a5d 100644 --- a/lib/agent/actions/fileretrieval/upload.js +++ b/lib/agent/actions/fileretrieval/upload.js @@ -1,7 +1,6 @@ #!/usr/bin/env node -var files = require('./storage'), - fs = require('fs'), +var fs = require('fs'), path = require('path'), mime = require('mime'), common = require('./../../common'), @@ -12,17 +11,18 @@ var config = common.config, host = config._values['control-panel'].host, url = protocol + '://' + host; -var UPLOAD_SERVER = url + '/upload/upload', +var UPLOAD_SERVER = url + '/upload/upload', RESUMABLE_HEADER = 'X-Prey-Upload-Resumable', - OPEN_TIMEOUT = 180000, - READ_TIMEOUT = 5000; + OPEN_TIMEOUT = 180000, + READ_TIMEOUT = 5000; -var PATH = 2, - USER = 3, - NAME = 4, - SIZE = 5, +var PATH = 2, + USER = 3, + NAME = 4, + SIZE = 5, FILE_ID = 6, - PORT = 7; + RESUME = 7, + PORT = 8; function main() { var argv = process.argv; @@ -32,6 +32,7 @@ function main() { name: argv[NAME], size: argv[SIZE], file_id: argv[FILE_ID], + resumable : argv[RESUME], port: argv[PORT] } Main(options, function(err) { @@ -39,7 +40,6 @@ function main() { console.error(err); return; } - files.del(options.file_id); }); } @@ -52,10 +52,6 @@ function Main(options, cb) { console.log("Uploading file: ", file_path, file_id); - if (!options.resumable) { - files.store(file_id, file_path, file_size, user, file_name); - } - var file = { total: 0, path: file_path, @@ -68,7 +64,8 @@ function Main(options, cb) { } function retrieve_file(file, cb) { - if (file.resumable) { + console.log("RETRIEVE FILE:", file) + if (file.resumable == 'true') { console.log("Resumable file:", file.id); var url = UPLOAD_SERVER + '?uploadID=' + file.id; // Make a call to get the last byte processed by the upload server @@ -79,6 +76,7 @@ function retrieve_file(file, cb) { return; } var data = JSON.parse(res.body); + file.total = data.Total; get_file(file, cb); }) @@ -106,7 +104,7 @@ function upload_file(file, buf, cb) { open_timeout: OPEN_TIMEOUT, read_timeout: READ_TIMEOUT }; - + if (file.total > 0) { RESUMABLE_HEADER = file.total; } From 89f922146ab00b9da76e8a345387975035a1a91e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Acun=CC=83a?= Date: Wed, 18 May 2016 11:36:15 -0300 Subject: [PATCH 3/8] Modify retry parameters --- lib/agent/actions/fileretrieval/index.js | 9 ++++----- lib/agent/actions/fileretrieval/storage.js | 2 +- lib/agent/actions/fileretrieval/upload.js | 9 +++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/agent/actions/fileretrieval/index.js b/lib/agent/actions/fileretrieval/index.js index 5178dd4cd..3e712b18c 100644 --- a/lib/agent/actions/fileretrieval/index.js +++ b/lib/agent/actions/fileretrieval/index.js @@ -52,12 +52,11 @@ exports.start = function(options, cb) { logger.error("Upload error: " + err.message); return; } - logger.info("Ran as user:" + out); - - files.del(options.file_id); - + logger.info("Ran as user: " + out); + if (out.indexOf("File succesfuly uploaded") != -1){ + files.del(options.file_id); + } }); - if (cb) cb(null, em); em.emit('end'); } diff --git a/lib/agent/actions/fileretrieval/storage.js b/lib/agent/actions/fileretrieval/storage.js index 1f87b7013..d9fe5dccc 100644 --- a/lib/agent/actions/fileretrieval/storage.js +++ b/lib/agent/actions/fileretrieval/storage.js @@ -16,7 +16,7 @@ var exist = function(id, cb) { storage.all(function(err, files) { if (err) return err.message; - if (files[key]) + if (files[key]) return cb(true); return cb(false); }); diff --git a/lib/agent/actions/fileretrieval/upload.js b/lib/agent/actions/fileretrieval/upload.js index 7dc0e3a5d..30b8baf98 100644 --- a/lib/agent/actions/fileretrieval/upload.js +++ b/lib/agent/actions/fileretrieval/upload.js @@ -14,7 +14,7 @@ var config = common.config, var UPLOAD_SERVER = url + '/upload/upload', RESUMABLE_HEADER = 'X-Prey-Upload-Resumable', OPEN_TIMEOUT = 180000, - READ_TIMEOUT = 5000; + READ_TIMEOUT = 2000; var PATH = 2, USER = 3, @@ -64,7 +64,6 @@ function Main(options, cb) { } function retrieve_file(file, cb) { - console.log("RETRIEVE FILE:", file) if (file.resumable == 'true') { console.log("Resumable file:", file.id); var url = UPLOAD_SERVER + '?uploadID=' + file.id; @@ -72,6 +71,7 @@ function retrieve_file(file, cb) { // in order to resume the upload from that position. needle.request('get', url, null, function(err, res) { if (err) { + console.log(err); cb(err); return; } @@ -103,15 +103,16 @@ function upload_file(file, buf, cb) { var options = { open_timeout: OPEN_TIMEOUT, read_timeout: READ_TIMEOUT - }; + } if (file.total > 0) { RESUMABLE_HEADER = file.total; } - var url = UPLOAD_SERVER + '?uploadID='+file.id; + var url = UPLOAD_SERVER + '?uploadID=' + file.id; needle.post(url, buf, options, function(err, res) { if (err) { + console.log("ERROR", err); cb(err); return; } From f8655634b1065876be85f2e9d924c6721fdc1f16 Mon Sep 17 00:00:00 2001 From: javo Date: Wed, 18 May 2016 12:52:09 -0300 Subject: [PATCH 4/8] Fix path and files name in windows --- lib/agent/actions/fileretrieval/index.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/agent/actions/fileretrieval/index.js b/lib/agent/actions/fileretrieval/index.js index 3e712b18c..1fe3eb925 100644 --- a/lib/agent/actions/fileretrieval/index.js +++ b/lib/agent/actions/fileretrieval/index.js @@ -19,6 +19,7 @@ var fs = require('fs'), var system = common.system, run_as_user = common.system.run_as_user, node_bin = path.join(system.paths.current, 'bin', 'node'), + os_name = process.platform.replace('darwin', 'mac').replace('win32', 'windows'), logger = common.logger; var em, @@ -34,11 +35,18 @@ exports.start = function(options, cb) { if (!options.resumable) { options.resumable = false; } + + var argsv; + if (os_name == 'windows') + argsv = [path.join(__dirname, 'upload.js'), path.resolve(options.path), options.user, path.resolve(options.name), options.size, options.file_id, options.resumable, options.port]; + else + argsv = [path.join(__dirname, 'upload.js'), '"' + options.path + '"', options.user, '"'+options.name+'"', options.size, options.file_id, options.resumable, options.port]; + var opts = { user: options.user, bin: node_bin, type: 'exec', - args: [path.join(__dirname, 'upload.js'), '"' + options.path + '"', options.user, '"'+options.name+'"', options.size, options.file_id, options.resumable, options.port], + args: argsv, opts: { env: process.env } From 4ccc9c3513b9b4aafdcf9e54d7acd88dd577f761 Mon Sep 17 00:00:00 2001 From: javo Date: Fri, 20 May 2016 10:50:35 -0300 Subject: [PATCH 5/8] Upload error response modified --- lib/agent/actions/fileretrieval/storage.js | 2 ++ lib/agent/actions/fileretrieval/upload.js | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/agent/actions/fileretrieval/storage.js b/lib/agent/actions/fileretrieval/storage.js index d9fe5dccc..97ae8c905 100644 --- a/lib/agent/actions/fileretrieval/storage.js +++ b/lib/agent/actions/fileretrieval/storage.js @@ -25,6 +25,7 @@ var exist = function(id, cb) { exports.store = function(id, path, size, user, name) { exist(id, function(cb) { if (cb == false) { + logger.debug('Storing file_id in DB: ' + id); var opts = { path: path, size: size, @@ -37,6 +38,7 @@ exports.store = function(id, path, size, user, name) { } exports.del = function(id) { + logger.debug('Removing file_id from DB: ' + id); storage.del(NAMESPACE + id); } diff --git a/lib/agent/actions/fileretrieval/upload.js b/lib/agent/actions/fileretrieval/upload.js index 30b8baf98..a8854198c 100644 --- a/lib/agent/actions/fileretrieval/upload.js +++ b/lib/agent/actions/fileretrieval/upload.js @@ -112,7 +112,7 @@ function upload_file(file, buf, cb) { needle.post(url, buf, options, function(err, res) { if (err) { - console.log("ERROR", err); + console.log(err); cb(err); return; } From d76065daa07ffbb6d8670b12687bfda7ada20862 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Acun=CC=83a?= Date: Mon, 23 May 2016 16:07:31 -0300 Subject: [PATCH 6/8] Reports url issue fixed --- lib/agent/plugins/control-panel/api/push.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/agent/plugins/control-panel/api/push.js b/lib/agent/plugins/control-panel/api/push.js index 8566ecc7c..76217d83f 100644 --- a/lib/agent/plugins/control-panel/api/push.js +++ b/lib/agent/plugins/control-panel/api/push.js @@ -38,7 +38,7 @@ exports.data = function(data, opts, cb) { exports.report = function(data, opts, cb) { check_keys(); - var url = format_url(reports); + var url = format_url('reports'); post(url, data, opts, cb); } From a0d82c017e44689b74b60490ea26d8e174bd6e4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Acun=CC=83a?= Date: Tue, 24 May 2016 16:51:37 -0300 Subject: [PATCH 7/8] Fix winattr on ubuntu --- lib/agent/providers/files/tree.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/agent/providers/files/tree.js b/lib/agent/providers/files/tree.js index 20134929f..780eab0c5 100644 --- a/lib/agent/providers/files/tree.js +++ b/lib/agent/providers/files/tree.js @@ -15,8 +15,7 @@ var fs = require('fs'), path = require('path'), - mime = require('mime'), - winattr = require('winattr'); + mime = require('mime'); var argv = process.argv, p = argv[3], @@ -110,6 +109,7 @@ directoryTreeToObj(p, cd, d, function(err, res) { var checkIfHiddenFile = function(path, name, cb){ if (os_name == "windows") { + winattr = require('winattr'); winattr.get(path, function(err, attrs){ if (err) return cb(false); From 0268cfe5ac1d4a49c53b54621f115c10ecca1cda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Acun=CC=83a?= Date: Tue, 31 May 2016 15:38:53 -0300 Subject: [PATCH 8/8] path and name arguments fix --- lib/agent/actions/fileretrieval/index.js | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/agent/actions/fileretrieval/index.js b/lib/agent/actions/fileretrieval/index.js index 1fe3eb925..f4c4da32d 100644 --- a/lib/agent/actions/fileretrieval/index.js +++ b/lib/agent/actions/fileretrieval/index.js @@ -25,6 +25,9 @@ var system = common.system, var em, cp; +var path_arg, + name_arg; + // check_pending_files is used to resume any files that might been pending. It's called from // filesagent/providers/network. exports.check_pending_files = function() { @@ -35,18 +38,18 @@ exports.start = function(options, cb) { if (!options.resumable) { options.resumable = false; } - - var argsv; - if (os_name == 'windows') - argsv = [path.join(__dirname, 'upload.js'), path.resolve(options.path), options.user, path.resolve(options.name), options.size, options.file_id, options.resumable, options.port]; - else - argsv = [path.join(__dirname, 'upload.js'), '"' + options.path + '"', options.user, '"'+options.name+'"', options.size, options.file_id, options.resumable, options.port]; - + if (os_name == 'windows') { + path_arg = path.resolve(options.path); + name_arg = path.resolve(options.name); + } else { + path_arg = '"' + options.path + '"'; + name_arg = '"' + options.name + '"'; + } var opts = { user: options.user, bin: node_bin, type: 'exec', - args: argsv, + args: [path.join(__dirname, 'upload.js'), path_arg, options.user, name_arg, options.size, options.file_id, options.resumable, options.port], opts: { env: process.env }