Releasing version 2.4.10 (#23)

CHANGELOG.rst

@@ -7,6 +7,30 @@ All notable changes to this project will be documented in this file.
 The format is based on `Keep a
 Changelog <http://keepachangelog.com/>`__.
 
+2.4.10 - 2017-10-12
+-------------------
+
+Added
+~~~~~~~~~~
+* Support for new Database service operations: VM DBs, Bring Your Own License, and Data Guard.
+* Support for autocomplete on Windows (PowerShell only)
+* Support for defaults file to specify default values for CLI parameters (https://github.com/oracle/oci-cli/issues/20)
+* Support for parallelization in bulk object storage commands: bulk upload / download / delete).
+* Support for including / excluding files in bulk upload / download / delete based on file patterns.
+* Support for enabling / disabling VNIC source/destination checks (https://github.com/oracle/oci-cli/issues/15)
+* Support for adding and updating display names for captured instance serial console data.
+* Display public key fingerprint in output of 'oci setup config' (https://github.com/oracle/oci-cli/issues/18)
+
+Fixed
+~~~~~~~~~~
+* Allow piping input through STDIN for 'oci os object put' (https://github.com/oracle/oci-cli/issues/21)
+* Use full path when writing 'key_file' in 'oci setup config' (https://github.com/oracle/oci-cli/issues/19)
+* Added missing files and instructions to allow running tests
+
+Deprecated
+~~~~~~~~~~
+* oci bv volume create --size-in-mbs parameter is now deprecated in favor of the new --size-in-gbs parameter
+
 2.4.9 - 2017-09-13
 ------------------
 

MANIFEST.in

@@ -2,4 +2,5 @@ include LICENSE.txt
 include README.rst
 include CHANGELOG.md
 include src/oci_cli/bin/oci_autocomplete.sh
+include src/oci_cli/bin/OciTabExpansion.ps1
 exclude setup.cfg

README.rst

@@ -13,13 +13,21 @@ __ https://docs.us-phoenix-1.oraclecloud.com/Content/API/SDKDocs/cli.htm
 Installation
 ============
 
+Mac / Linux
+-----------
 ::
 
-    pip install oci-cli
+    curl -L "https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh" | bash
 
-See the `installation guide`__ for installation options and installation troubleshooting.
+Windows
+-------
+::
+
+    powershell -NoProfile -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.ps1'))"
+
+See the `installation guide`__ for detailed installation instructions, options and troubleshooting.
 
-__ https://docs.us-phoenix-1.oraclecloud.com/Content/API/SDKDocs/cli.htm#cli-install
+__ https://docs.us-phoenix-1.oraclecloud.com/Content/API/SDKDocs/cli.htm#CLIInstallationOptions
 
 
 Usage

requirements.txt

@@ -1,4 +1,5 @@
 appdirs==1.4.3
+arrow==0.10.0
 asn1crypto==0.22.0
 certifi==2017.1.23
 cffi==1.9.1
@@ -7,9 +8,11 @@ configparser==3.5.0
 cryptography==1.8.2
 httpsig-cffi==15.0.0
 idna==2.5
+Jinja2==2.9.6
+jmespath==0.9.3
 ndg-httpsclient==0.4.2
 mock==2.0.0
-oci==1.3.7
+oci==1.3.8
 packaging==16.8
 pluggy==0.4.0
 py==1.4.32
@@ -25,4 +28,4 @@ retrying==1.3.3
 six==1.10.0
 terminaltables==3.1.0
 tox==2.6.0
-virtualenv==15.1.0
+virtualenv==15.1.0

scripts/basic_cli_test_script.sh

@@ -10,6 +10,8 @@ BUCKET=TestScriptBucket$(( ( RANDOM % 10000 )  + 1 ))
 C=$OCI_CLI_COMPARTMENT_ID
 FILE=scripts/temp/content.txt
 EMPTY_FILE=scripts/temp/empty_file
+LARGE_FILE_PATH=scripts/temp/large_file
+LARGE_FILE_DOWNLOAD_PATH=scripts/temp/large_file_downloaded
 ARGS="--config-file $OCI_CLI_CONFIG_FILE"
 
 # test that invoking with deprecated entry point 'bmcs' still works
@@ -30,12 +32,40 @@ touch $EMPTY_FILE
 oci $ARGS os object put -ns $NS -bn $BUCKET --name object3 --file $EMPTY_FILE
 oci $ARGS os object get -ns $NS -bn $BUCKET --name object3 --file -
 
-# Put a file from stdin and get it back. Also check we can assign metadata when putting from stdin
+# Put a file from stdin with a here string and get it back. Also check we can assign metadata when putting from stdin
 oci $ARGS os object put -ns $NS -bn $BUCKET --name object4 --metadata '{"foo1":"bar1","foo2":"bar2"}' --file - <<< "This is some object content"
 oci $ARGS os object get -ns $NS -bn $BUCKET --name object4 --file - | grep "This is some object content"
 oci $ARGS os object head -ns $NS -bn $BUCKET --name object4 | grep foo1
 oci $ARGS os object head -ns $NS -bn $BUCKET --name object4 | grep bar2
 
+# Put an empty file from stdin when piping and get it back
+cat $EMPTY_FILE | oci $ARGS os object put -ns $NS -bn $BUCKET --name object5 --file -
+oci $ARGS os object get -ns $NS -bn $BUCKET --name object5 --file -
+
+# Put a file from stdin with piping and get it back. Also check we can assign metadata when piping
+echo "I am the very model of a modern Major General" | oci $ARGS os object put -ns $NS -bn $BUCKET --name object5 --metadata '{"foo1":"bar1","foo2":"bar2"}' --file - --force
+oci $ARGS os object get -ns $NS -bn $BUCKET --name object5 --file - | grep "I am the very model of a modern Major General"
+oci $ARGS os object head -ns $NS -bn $BUCKET --name object5 | grep foo1
+oci $ARGS os object head -ns $NS -bn $BUCKET --name object5 | grep bar2
+
+# Put a large file into object storage, then get it and pipe it to another put.
+# The large file is 50MiB (12,800 4k blocks)
+dd if=/dev/zero of=$LARGE_FILE_PATH bs=4k count=12800
+oci $ARGS os object put -ns $NS -bn $BUCKET --name object6 --file $LARGE_FILE_PATH
+oci $ARGS os object get -ns $NS -bn $BUCKET --name object6 --file - | oci $ARGS os object put -ns $NS -bn $BUCKET --name object7 --file - 
+oci $ARGS os object get -ns $NS -bn $BUCKET --name object7 --file $LARGE_FILE_DOWNLOAD_PATH
+
+# Belt and suspenders (may be overkill) - compare the file and the MD5
+cmp $LARGE_FILE_PATH $LARGE_FILE_DOWNLOAD_PATH
+ORIGINAL_MD5=$(openssl md5 $LARGE_FILE_PATH | cut -d ' ' -f2)
+FINAL_MD5=$(openssl md5 $LARGE_FILE_DOWNLOAD_PATH | cut -d ' ' -f2)
+if [[ $FINAL_MD5 == $ORIGINAL_MD5 ]]; then
+    echo "MD5 validated."
+else
+    echo "MD5 does not match."
+    exit 1
+fi
+
 # Try a few variations on metadata format.
 oci $ARGS os object put -ns $NS -bn $BUCKET --name object2 --file $FILE --metadata '{"foo1":"a b c '"'d'"'","foo2": "bar2"}' --force
 oci $ARGS os object head -ns $NS -bn $BUCKET --name object2 | grep "a b c 'd'"
@@ -69,6 +99,9 @@ oci $ARGS os object delete -ns $NS -bn $BUCKET --name object1 --force
 oci $ARGS os object delete -ns $NS -bn $BUCKET --name object2 --force
 oci $ARGS os object delete -ns $NS -bn $BUCKET --name object3 --force
 oci $ARGS os object delete -ns $NS -bn $BUCKET --name object4 --force
+oci $ARGS os object delete -ns $NS -bn $BUCKET --name object5 --force
+oci $ARGS os object delete -ns $NS -bn $BUCKET --name object6 --force
+oci $ARGS os object delete -ns $NS -bn $BUCKET --name object7 --force
 oci $ARGS os bucket delete -ns $NS --name $BUCKET --force
 
 echo "Success!"

scripts/install/install.ps1

@@ -24,7 +24,7 @@ $OriginalErrorActionPreference = $ErrorActionPreference
 # Explicitly support TLS 1.2
 [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]'Ssl3,Tls,Tls11,Tls12';
 
-$PythonInstallScriptUrl = "https://raw.githubusercontent.com/oracle/oci-cli/1e2df8f61fcf85fcda9e9938365e76fbbece0ca3/scripts/install/install.py"
+$PythonInstallScriptUrl = "https://raw.githubusercontent.com/oracle/oci-cli/v2.4.10/scripts/install/install.py"
 $PythonVersionToInstall = "3.6.2"    # version of Python to install if none exists
 $MinValidPython2Version = "2.7.5"    # minimum required version of Python 2 on system
 $MinValidPython3Version = "3.5.0"    # minimum required version of Python 3 on system

scripts/install/install.py

@@ -8,6 +8,7 @@
 from __future__ import print_function
 import argparse
 import os
+import os.path
 import sys
 import platform
 import stat
@@ -31,14 +32,16 @@
     # Python 3 doesn't have raw_input
     pass
 
+
 def is_windows():
     return sys.platform == 'win32'
 
+
 ACCEPT_ALL_DEFAULTS = False
 
 VIRTUALENV_VERSION = '15.0.0'
-VIRTUALENV_ARCHIVE = 'virtualenv-'+VIRTUALENV_VERSION+'.tar.gz'
-VIRTUALENV_DOWNLOAD_URL = 'https://pypi.python.org/packages/source/v/virtualenv/'+VIRTUALENV_ARCHIVE
+VIRTUALENV_ARCHIVE = 'virtualenv-' + VIRTUALENV_VERSION + '.tar.gz'
+VIRTUALENV_DOWNLOAD_URL = 'https://pypi.python.org/packages/source/v/virtualenv/' + VIRTUALENV_ARCHIVE
 VIRTUALENV_ARCHIVE_SHA256 = '70d63fb7e949d07aeb37f6ecc94e8b60671edb15b890aa86dba5dfaf2225dc19'
 
 DEFAULT_INSTALL_DIR = os.path.expanduser(os.path.join('~', 'lib', 'oracle-cli'))
@@ -51,18 +54,19 @@ def is_windows():
 USER_BASH_PROFILE = os.path.expanduser(os.path.join('~', '.bash_profile'))
 SOURCE_AUTOCOMPLETE_COMMAND_TEMPLATE = '[[ -e "{completion_file_path}" ]] && source "{completion_file_path}"'
 RELATIVE_PATH_TO_AUTOCOMPLETE_SCRIPT = os.path.join('oci_cli', 'bin', 'oci_autocomplete.sh')
+RELATIVE_PATH_TO_POWERSHELL_AUTOCOMPLETE_SCRIPT = os.path.join('oci_cli', 'bin', 'OciTabExpansion.ps1')
 
 
 class CLIInstallError(Exception):
     pass
 
 
 def print_status(msg=''):
-    print('-- '+msg)
+    print('-- ' + msg)
 
 
 def prompt_input(msg):
-    return input('\n===> '+msg)
+    return input('\n===> ' + msg)
 
 
 def prompt_input_with_default(msg, default):
@@ -95,7 +99,7 @@ def prompt_y_n(msg, default=None):
 
 
 def exec_command(command_list, cwd=None, env=None):
-    print_status('Executing: '+str(command_list))
+    print_status('Executing: ' + str(command_list))
     subprocess.check_call(command_list, cwd=cwd, env=env)
 
 
@@ -122,7 +126,8 @@ def create_virtualenv(tmp_dir, install_dir):
     download_location = os.path.join(tmp_dir, VIRTUALENV_ARCHIVE)
     print_status('Downloading virtualenv package from {}.'.format(VIRTUALENV_DOWNLOAD_URL))
     response = urlopen(VIRTUALENV_DOWNLOAD_URL)
-    with open(download_location, 'wb') as f: f.write(response.read())
+    with open(download_location, 'wb') as f:
+        f.write(response.read())
     print_status("Downloaded virtualenv package to {}.".format(download_location))
     if is_valid_sha256sum(download_location, VIRTUALENV_ARCHIVE_SHA256):
         print_status("Checksum of {} OK.".format(download_location))
@@ -132,10 +137,10 @@ def create_virtualenv(tmp_dir, install_dir):
     package_tar = tarfile.open(download_location)
     package_tar.extractall(path=tmp_dir)
     package_tar.close()
-    virtualenv_dir_name = 'virtualenv-'+VIRTUALENV_VERSION
+    virtualenv_dir_name = 'virtualenv-' + VIRTUALENV_VERSION
     working_dir = os.path.join(tmp_dir, virtualenv_dir_name)
 
-    # due to an issue with virtualenv on windows, we need to explicitly copy some dlls into the virtual environment 
+    # due to an issue with virtualenv on windows, we need to explicitly copy some dlls into the virtual environment
     # or the python executable in the virtualenv will crash upon invocation
     # for python 3 one possible alternative is to use venv instead of virtualenv
     if is_windows():
@@ -146,7 +151,7 @@ def create_virtualenv(tmp_dir, install_dir):
         for dll in glob.glob(os.path.join(src_dir, '*.dll')):
             print_status('Copying {} to {}'.format(dll, dest_dir))
             shutil.copy(dll, dest_dir)
-    
+
     cmd = [sys.executable, 'virtualenv.py', '--python', sys.executable, install_dir]
     exec_command(cmd, cwd=working_dir)
 
@@ -177,7 +182,7 @@ def get_install_dir():
                 if ans_yes:
                     try:
                         shutil.rmtree(install_dir)
-                    except Exception as e:
+                    except Exception:
                         sys.exit("Failed to remove directory: {}. Please remove directory manually and re-run installation script.".format(install_dir))
 
                     print_status("Deleted '{}'.".format(install_dir))
@@ -205,8 +210,8 @@ def get_exec_dir():
 
 def _backup_rc(rc_file):
     try:
-        shutil.copyfile(rc_file, rc_file+'.backup')
-        print_status("Backed up '{}' to '{}'".format(rc_file, rc_file+'.backup'))
+        shutil.copyfile(rc_file, rc_file + '.backup')
+        print_status("Backed up '{}' to '{}'".format(rc_file, rc_file + '.backup'))
     except (OSError, IOError):
         pass
 
@@ -244,7 +249,7 @@ def _find_line_in_file(file_path, search_pattern):
 def _modify_rc(rc_file_path, line_to_add):
     if not _find_line_in_file(rc_file_path, line_to_add):
         with open(rc_file_path, 'a') as rc_file:
-            rc_file.write('\n'+line_to_add+'\n')
+            rc_file.write('\n' + line_to_add + '\n')
 
 
 def get_rc_file_path():
@@ -282,8 +287,35 @@ def warn_other_oci_or_bmcs_on_path(exec_dir, exec_filepath):
 
 def handle_path_and_tab_completion(completion_file_path, exec_filepath, exec_dir):
     if is_windows():
-        ans_yes = prompt_y_n('Modify PATH to include the CLI?', 'y')
+        ans_yes = prompt_y_n('Modify PATH to include the CLI and enable tab completion in PowerShell now?', 'y')
         if ans_yes:
+            # Add autocomplete to the user's profile. Assume that powershell is on the path
+            profile_file_path = subprocess.check_output(['powershell', '-NoProfile', 'echo $profile']).decode(sys.stdout.encoding).strip()
+            if not os.path.exists(profile_file_path):
+                if not os.path.exists(os.path.dirname(profile_file_path)):
+                    os.makedirs(os.path.dirname(profile_file_path))
+
+                with open(profile_file_path, 'w') as f:
+                    f.write('. {}'.format(completion_file_path))
+            else:
+                with open(profile_file_path, 'r') as f:
+                    current_file_contents = f.read()
+
+                if current_file_contents.find(completion_file_path) >= 0:
+                    # They have the tab completion script in the place we thought already. No action needed
+                    pass
+                elif current_file_contents.find('OciTabExpansion.ps1') >= 0:
+                    # They have the tab completion script, but not in the place we thought. Print out a warning
+                    # but otherwise take no action on the profile
+                    print_status()
+
+                    format_str = "It looks like tab completion for oci is already configured in {profile_file_path}. If you wish to replace this with the tab completion script included in this version of the CLI, please remove any lines containing 'OciTabExpansion.ps1' from {profile_file_path} and then run 'oci setup autocomplete'"
+                    print_status(format_str.format(profile_file_path=profile_file_path))
+                else:
+                    # They don't have the tab completion script in their profile. Add it
+                    with open(profile_file_path, 'a'):
+                        f.write('\n. {}\n'.format(completion_file_path))
+
             # powershell one-liner to append the exec_dir to the USER path permanently
             # makes the assumption that powershell is on the PATH already
             command = "powershell -Command \"[Environment]::SetEnvironmentVariable(\\\"PATH\\\", \\\"{};\\\" + (Get-ItemProperty -Path 'Registry::HKEY_CURRENT_USER\Environment' -Name PATH).Path, \\\"User\\\")".format(exec_dir)
@@ -292,6 +324,7 @@ def handle_path_and_tab_completion(completion_file_path, exec_filepath, exec_dir
             print_status('** Close and re-open powershell to reload changes to your PATH **')
             print_status()
         else:
+            print_status("If you change your mind, dot source {} in your PowerShell profile and restart your shell to enable tab completion.".format(completion_file_path))
             print_status("You can run the CLI with '{}'.".format(exec_filepath))
     else:
         ans_yes = prompt_y_n('Modify profile to update your $PATH and enable shell/tab completion now?', 'y')
@@ -408,14 +441,15 @@ def main():
     install_cli(install_dir, tmp_dir)
 
     if is_windows():
-        # tab completion is not supported on windows
-        completion_file_path = None
-        # copy the executable created from the pip install to the bin directory specified by the user 
+        venv_python_executable = os.path.join(install_dir, 'Scripts', 'python')
+        venv_site_packages_directory = subprocess.check_output([venv_python_executable, '-c', 'from distutils.sysconfig import get_python_lib; print(get_python_lib())']).strip()
+        completion_file_path = os.path.join(venv_site_packages_directory.decode(sys.stdout.encoding), RELATIVE_PATH_TO_POWERSHELL_AUTOCOMPLETE_SCRIPT)
+
+        # copy the executable created from the pip install to the bin directory specified by the user
         shutil.copyfile(os.path.join(install_dir, 'Scripts', OCI_EXECUTABLE_NAME), oci_exec_path)
         shutil.copyfile(os.path.join(install_dir, 'Scripts', BMCS_EXECUTABLE_NAME), bmcs_exec_path)
     else:
-        original_exec_filepath = os.path.join(install_dir, 'bin', OCI_EXECUTABLE_NAME)
-        # copy the executable created from the pip install to the bin directory specified by the user 
+        # copy the executable created from the pip install to the bin directory specified by the user
         shutil.copyfile(os.path.join(install_dir, 'bin', OCI_EXECUTABLE_NAME), oci_exec_path)
         shutil.copyfile(os.path.join(install_dir, 'bin', BMCS_EXECUTABLE_NAME), bmcs_exec_path)
         oci_exec_cur_stat = os.stat(oci_exec_path)
@@ -439,12 +473,13 @@ def main():
     print_status("Installation successful.")
     print_status("Run the CLI with {} --help".format(oci_exec_path))
 
+
 if __name__ == '__main__':
     try:
         main()
     except CLIInstallError as cie:
-        print('ERROR: '+str(cie), file=sys.stderr)
+        print('ERROR: ' + str(cie), file=sys.stderr)
         sys.exit(1)
     except KeyboardInterrupt:
         print('\n\nExiting...')
-        sys.exit(1)
+        sys.exit(1)

scripts/install/install.sh

@@ -4,7 +4,7 @@
 # Bash script to install the Oracle Cloud Infrastructure CLI
 # Example invocation: curl -L "https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh" | bash
 #
-INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/oracle/oci-cli/1e2df8f61fcf85fcda9e9938365e76fbbece0ca3/scripts/install/install.py"
+INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/oracle/oci-cli/v2.4.10/scripts/install/install.py"
 _TTY=/dev/tty
 
 install_script=$(mktemp -t oci_cli_install_tmp_XXXX) || exit

setup.py

@@ -30,13 +30,15 @@ def open_relative(*path):
 
 
 requires = [
-    'oci==1.3.7',
+    'oci==1.3.8',
+    'arrow==0.10.0',
     'certifi',
     'click==6.7',
     'configparser==3.5.0',
     'cryptography==1.8.2',
     'pyOpenSSL<=17.0.0',
     'httpsig_cffi==15.0.0',
+    'jmespath==0.9.3',
     'python-dateutil==2.5.3',
     'pytz==2016.7',
     'retrying==1.3.3',