Fixing Issue marklogic-community#521: support custom properties for C…

…oRB2

Updated the corb task to use CoRB2 v2.3.1 and support all of the CoRB2 options.

Ensured that the current Roxy commandline options and functionality still work,
and also support specifying any of the CoRB2 options with either "--" or "-D"
prefix, case-insensitive (i.e. --options-file or -DOPTIONS-FILE).

Removed the explicit checks/errors for Roxy corb task required fields, since there
are now additional options available: URIS-FILE, XML-FILE, PROCESS-MODULE (now 
preferred instead of XQUERY-MODULE), specifying all options inside of 
OPTIONS-FILE, etc.

Also upgraded the XCC jar to v8.0.5

deploy/lib/Help.rb

@@ -398,30 +398,27 @@ def self.corb
     <<-DOC.strip_heredoc
       Usage: ml {env} corb [options]
 
-      See: http://marklogic.github.com/corb/index.html
+      Runs CoRB2 with given command-line options against the selected environment.
+      CoRB2 options may be specified on the commandline using either -- or -D prefix.
 
-      Required options:
-        --modules=/path/to/modules.xqy  # the xquery module to process the data
+      For example, the OPTIONS-FILE option can be specified as:
+        --options-file=/path/to/options.properties
+        -DOPTIONS-FILE=/path/to/options.properties
 
-        (Only one of the following is required)
-        --collection=collection-name    # the name of a collection to process
-        --uris=/path/to/uris-module.xqy # path to a uris module
+      CoRB2 supports options files natively using the --OPTIONS-FILE parameter.
 
-      Corb Options:
-        --threads=1                     # the thread count to use
-        --root=/                        # the root of the modules database
-        --install=false                 # whether or not to install (default: false)
+      See: https://github.com/marklogic/corb2
 
       General options:
-        -v, [--verbose]  # Verbose output
+        -h, [--help]     # Shows this help
     DOC
   end
 
   def self.mlcp
     <<-DOC.strip_heredoc
       Usage: ml {env} mlcp [options]
 
-      Runs MLCP with given command-line options agains selected environment.
+      Runs MLCP with given command-line options against selected environment.
       MLCP supports options files natively using the -options_file parameter.
       The path to the MLCP options file must be an absolute path or a relative
       path from the root of the project directory.

deploy/lib/server_config.rb

@@ -1142,38 +1142,150 @@ def corb
     password_prompt
     encoded_password = url_encode(@ml_password)
     connection_string = %Q{xcc://#{@properties['ml.user']}:#{encoded_password}@#{@properties['ml.server']}:#{@properties['ml.xcc-port']}/#{@properties['ml.content-db']}}
-    collection_name = find_arg(['--collection']) || '""'
-    xquery_module = find_arg(['--modules'])
-    uris_module = find_arg(['--uris']) || '""'
 
+    options = Hash.new("")
+    # handle Roxy convention for CoRB properties first
+    process_module = find_arg(['--modules']) || ''
+    process_module = process_module.reverse.chomp("/").reverse
+    if !process_module.blank?
+        options["PROCESS-MODULE"] = process_module
+    end
+
+    collection_name = find_arg(['--collection']) || ''
+    if !collection_name.blank?
+        options["COLLECTION-NAME"] = collection_name
+        # when COLLECTION-NAME is specified, assume CoRB 1.0 convention,
+        # and set URIS-MODULE with an inline module to return the URIs of all docs in the specified collection(s)
+        options["URIS-MODULE"] = "INLINE-XQUERY|xquery version '1.0-ml'; declare variable \\$URIS as xs:string external; let \\$uris := cts:uris('', ('document'), cts:collection-query(\\$URIS)) return (count(\\$uris), \\$uris)"
+    end
 
-    raise HelpException.new("corb", "modules is required") if xquery_module.blank?
-    raise HelpException.new("corb", "uris or collection is required ") if uris_module == '""' && collection_name == '""'
+    uris_module = find_arg(['--uris']) || ''
+    uris_module = uris_module.reverse.chomp('/').reverse
+    if !uris_module.blank?
+        options["URIS-MODULE"] = uris_module
+    end
 
-    xquery_module = xquery_module.reverse.chomp("/").reverse
-    uris_module = uris_module.reverse.chomp("/").reverse
-    thread_count = find_arg(['--threads']) || "1"
+    thread_count = find_arg(['--threads'])
     thread_count = thread_count.to_i
-    module_root = find_arg(['--root']) || '"/"'
-    modules_database = @properties['ml.modules-db']
-    install = find_arg(['--install']) == "true" || uris_module == '""'
+    if thread_count > 0
+        options["THREAD-COUNT"] = thread_count
+    end
+
+    module_root = find_arg(['--root']) || ''
+    if !module_root.blank?
+        options["MODULE-ROOT"] = module_root
+    end
+
+    modules_database = @properties['ml.modules-db'] || ''
+    if !modules_database.blank?
+        options["MODULES-DATABASE"] = modules_database
+    end
+
+    # A full list of CoRB2 options
+    ['BATCH-SIZE',
+    'BATCH-URI-DELIM',
+    'COLLECTION-NAME',
+    'COMMAND',
+    'COMMAND-FILE',
+    'COMMAND-FILE-POLL-INTERVAL',
+    'DECRYPTER',
+    'DISK-QUEUE',
+    'DISK-QUEUE-TEMP-DIR',
+    'DISK-QUEUE-MAX-IN-MEMORY-SIZE',
+    'ERROR-FILE-NAME',
+    'EXIT-CODE-NO-URIS',
+    'EXPORT_FILE_AS_ZIP',
+    'EXPORT-FILE-BOTTOM-CONTENT',
+    'EXPORT-FILE-DIR',
+    'EXPORT-FILE-HEADER-LINE-COUNT',
+    'EXPORT-FILE-NAME',
+    'EXPORT-FILE-PART-EXT',
+    'EXPORT-FILE-SORT',
+    'EXPORT-FILE-SORT-COMPARATOR',
+    'EXPORT-FILE-TOP-CONTENT',
+    'EXPORT-FILE-URI-TO-PATH',
+    'FAIL-ON-ERROR',
+    'INIT-MODULE',
+    'INIT-TASK',
+    'INSTALL',
+    'JASYPT-PROPERTIES-FILE',
+    'MAX_OPTS_FROM_MODULE',
+    'MODULES-DATABASE',
+    'MODULE-ROOT',
+    'OPTIONS-FILE',
+    'POST-BATCH-MODULE',
+    'POST-BATCH-TASK',
+    'POST-BATCH-XQUERY-MODULE',
+    'PRE-BATCH-MODULE',
+    'PRE-BATCH-TASK',
+    'PRE-BATCH-XQUERY-MODULE',
+    'PRIVATE-KEY-ALGORITHM',
+    'PRIVATE-KEY-FILE',
+    'PROCESS-MODULE',
+    'PROCESS-TASK',
+    'QUERY-RETRY-ERROR-CODES',
+    'QUERY-RETRY-ERROR-MESSAGE',
+    'QUERY-RETRY-INTERVAL',
+    'QUERY-RETRY-LIMIT',
+    'SSL-CIPHER-SUITES',
+    'SSL-CONFIG-CLASS',
+    'SSL-ENABLED-PROTOCOLS',
+    'SSL-KEY-PASSWORD',
+    'SSL-KEYSTORE',
+    'SSL-KEYSTORE-PASSWORD',
+    'SSL-KEYSTORE-TYPE',
+    'SSL-PROPERTIES-FILE',
+    'THREAD-COUNT',
+    'URIS_BATCH_REF',
+    'URIS-FILE',
+    'URIS-LOADER',
+    'URIS-MODULE',
+    'URIS-REPLACE-PATTERN',
+    'XCC-CONNECTION-RETRY-LIMIT',
+    'XCC-CONNECTION-RETRY-INTERVAL',
+    'XCC-CONNECTION-URI',
+    'XCC-DBNAME',
+    'XCC-HOSTNAME',
+    'XCC-PASSWORD',
+    'XCC-PORT',
+    'XCC-USERNAME',
+    'XML-FILE',
+    'XML-NODE',
+    'XQUERY-MODULE'].each do |optionName|
+        # match whether - or _ was used, since CoRB2 is not consistent with all option names
+        optionNamePattern = optionName.gsub(/(-|_)/, '\1?')
+        # match (case-insensitive) CoRB options with either "--" or "-D" prefix
+        optionArgPattern = /^(--|-D)(#{optionNamePattern})="?(.*)"?/i
+        ARGV.each do |arg|
+          if arg.match(optionArgPattern)
+            index = ARGV.index(arg)
+            ARGV.slice!(index)
+            matches = arg.match(optionArgPattern).to_a
+            options[optionName] = matches[3]
+          end
+        end
+    end
+
+    # collect options and set as Java system properties switches
+    systemProperties = options.delete_if{ |key, value| value.blank? }
+                        .map{ |key, value| "-D#{key}=\"#{value}\""}
+                        .join(' ')
 
     # Find the jars
-    corb_file = find_jar("corb")
-    xcc_file = find_jar("xcc")
+    corb_file = find_jar('corb')
+    xcc_file = find_jar('xcc')
+
+    runme = %Q{java -cp #{corb_file}#{path_separator}#{xcc_file} #{systemProperties} com.marklogic.developer.corb.Manager #{connection_string}}
+    logger.info runme
 
-    if install
+    if options.fetch("INSTALL", false)
       # If we're installing, we need to change directories to the source
       # directory, so that the xquery_modules will be visible with the
       # same path that will be used to see it in the modules database.
       Dir.chdir(@properties['ml.xquery.dir']) do
-        runme = %Q{java -cp #{corb_file}#{path_separator}#{xcc_file} com.marklogic.developer.corb.Manager #{connection_string} #{collection_name} #{xquery_module} #{thread_count} #{uris_module} #{module_root} #{modules_database} #{install}}
-        logger.info runme
         `#{runme}`
       end
     else
-      runme = %Q{java -cp #{corb_file}#{path_separator}#{xcc_file} com.marklogic.developer.corb.Manager #{connection_string} #{collection_name} #{xquery_module} #{thread_count} #{uris_module} #{module_root} #{modules_database} #{install}}
-      logger.info runme
       `#{runme}`
     end
   end