From 38f80b57ddad90d727e830d8daace14f342c3a37 Mon Sep 17 00:00:00 2001 From: sashakames Date: Wed, 2 Oct 2019 16:23:40 -0700 Subject: [PATCH 01/12] update version tags --- group_vars/index.yml | 2 +- roles/publisher/files/environment.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/group_vars/index.yml b/group_vars/index.yml index a1767ebd..12ca4cc1 100644 --- a/group_vars/index.yml +++ b/group_vars/index.yml @@ -18,7 +18,7 @@ solr: cog: repo: https://github.com/EarthSystemCoG/COG.git - version: v3.14.3 + version: v3.15.1 dest: /usr/local/cog/cog_install base: /usr/local/cog wsgi_dir: /etc/cog-wsgi-8889 diff --git a/roles/publisher/files/environment.yml b/roles/publisher/files/environment.yml index 83f746ce..8d8f3454 100644 --- a/roles/publisher/files/environment.yml +++ b/roles/publisher/files/environment.yml @@ -13,7 +13,7 @@ dependencies: - certifi=2019.3.9 - cffi=1.12.2 - chardet=3.0.4 - - cmor=3.4.0 + - cmor=3.5.0 - cryptography=2.6.1 - curl=7.64.0 - decorator=4.4.0 @@ -83,11 +83,11 @@ dependencies: - cdms2==3.0.0 - cf-python==2.3.3 - cftime==1.0.3.4 - - esgcet==3.7.0 + - esgcet==3.7.1 - esgconfigparser==0.1.17 - esgf-pyclient==0.2.1 - esgfpid==0.7.12 - - esgprep==2.9.5 + - esgprep==2.9.7 - fuzzywuzzy==0.16.0 - hurry-filesize==0.9 - idna==2.7 From 5ce49cb39a96f922f734830d53d11807b3e40d6a Mon Sep 17 00:00:00 2001 From: Nathan Carlson Date: Thu, 3 Oct 2019 16:17:46 -0700 Subject: [PATCH 02/12] Remove deployment of deprecated globusonline library With the release of CoG v3.15.1 comes an update to the latest Globus SDK library. --- roles/cog/tasks/main.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/roles/cog/tasks/main.yml b/roles/cog/tasks/main.yml index 9cc5902d..10bafcff 100644 --- a/roles/cog/tasks/main.yml +++ b/roles/cog/tasks/main.yml @@ -25,19 +25,6 @@ args: chdir: "{{ cog.dest }}" -# Install mkproxy -- name: Clone Transfer API Client Repo - git: - repo: "https://github.com/globusonline/transfer-api-client-python.git" - dest: "{{ cog.base }}/transfer-api-client-python" - -- name: Make and Install mkproxy - shell: > - {{ conda.actv }} cog && \ - make && make install - args: - chdir: "{{ cog.base }}/transfer-api-client-python/mkproxy" - - name: Install CoG into conda env shell: > {{ conda.actv }} cog && \ From 990f00eaf9ba634c6a9a58f4798634a24fe7801a Mon Sep 17 00:00:00 2001 From: Nathan Carlson Date: Fri, 4 Oct 2019 08:42:18 -0700 Subject: [PATCH 03/12] Properly resolve dependencies for esgf-pub env --- roles/publisher/files/environment.yml | 119 +++++++++++--------------- 1 file changed, 48 insertions(+), 71 deletions(-) diff --git a/roles/publisher/files/environment.yml b/roles/publisher/files/environment.yml index 8d8f3454..dafdc879 100644 --- a/roles/publisher/files/environment.yml +++ b/roles/publisher/files/environment.yml @@ -3,110 +3,87 @@ channels: - defaults - conda-forge dependencies: - - asn1crypto=0.24.0 - - attrs=19.1.0 - - blas=1.1 - - bzip2=1.0.6 - - ca-certificates=2019.1.23 - - cdat_info=8.1.1 - - cdtime=3.1.2 - - certifi=2019.3.9 - - cffi=1.12.2 - - chardet=3.0.4 + - _libgcc_mutex=0.1 + - blas=1.0 + - bzip2=1.0.8 + - ca-certificates=2019.8.28 + - certifi=2019.9.11 + - cftime=1.0.3.4 - cmor=3.5.0 - - cryptography=2.6.1 - - curl=7.64.0 - - decorator=4.4.0 - - distarray=2.12.2 - - enum34=1.1.6 - - esmf=7.1.0 - - esmpy=7.1.0 + - curl=7.65.3 - expat=2.2.6 - - functools32=3.2.3.2 - - future=0.17.1 - - g2clib=1.6.0 - hdf4=4.2.13 - - hdf5=1.10.4 - - ipaddress=1.0.22 - - ipython_genutils=0.2.0 - - jasper=1.900.1 + - hdf5=1.10.5 + - intel-openmp=2019.4 - jpeg=9c - - jsonschema=3.0.1 - - jupyter_core=4.4.0 + - json-c=0.13.1 - krb5=1.16.1 - - libcdms=3.1.2 - - libcf=1.0.2 - - libcurl=7.64.0 - - libdrs=3.1.2 - - libdrs_f=3.1.2 + - libcurl=7.65.3 - libedit=3.1.20181209 - libffi=3.2.1 - - libgcc-ng=8.2.0 - - libgfortran=3.0.0 + - libgcc-ng=9.1.0 - libgfortran-ng=7.3.0 - libnetcdf=4.6.2 - - libopenblas=0.2.20 - - libpng=1.6.36 - - libssh2=1.8.0 - - libstdcxx-ng=8.2.0 - - libtiff=4.0.10 - - mpi=1.0 - - mpich=3.2.1 - - nbformat=4.4.0 + - libssh2=1.8.2 + - libstdcxx-ng=9.1.0 + - libuuid=2.32.1 + - mkl=2019.4 + - mkl-service=2.3.0 + - mkl_fft=1.0.14 + - mkl_random=1.1.0 - ncurses=6.1 - - netcdf-fortran=4.4.5 - - numpy=1.15.2 - - openblas=0.2.20 - - openblas-devel=0.2.20 - - openssl=1.1.1b - - ossuuid=1.6.2 - - pip=19.0.3 - - pycparser=2.19 - - pyopenssl=19.0.0 - - pyrsistent=0.14.11 - - pysocks=1.6.8 + - numpy=1.16.5 + - numpy-base=1.16.5 + - openssl=1.1.1d + - pip=19.2.3 - python=2.7.16 - readline=7.0 - - setuptools=40.8.0 + - setuptools=41.2.0 - six=1.12.0 - - sqlite=3.27.2 + - sqlite=3.29.0 - tk=8.6.8 - - traitlets=4.3.2 - - udunits2=2.2.27 - - urllib3=1.24.1 - - wheel=0.33.1 - - xz=5.2.4 + - udunits2=2.2.25 + - wheel=0.33.6 - zlib=1.2.11 - - zstd=1.3.7 - pip: + - asn1crypto==1.0.0 - cdf2cim==0.3.3.0 - - cdms2==3.0.0 - - cf-python==2.3.3 - - cftime==1.0.3.4 + - cf-python==3.0.1 + - cfdm==1.7.8 + - cffi==1.12.3 + - cfunits==3.2.2 + - chardet==3.0.4 + - cryptography==2.7 + - decorator==4.4.0 + - enum34==1.1.6 - esgcet==3.7.1 - esgconfigparser==0.1.17 - esgf-pyclient==0.2.1 - - esgfpid==0.7.12 + - esgfpid==0.7.14 - esgprep==2.9.7 + - future==0.17.1 - fuzzywuzzy==0.16.0 - hurry-filesize==0.9 - idna==2.7 - - jinja2==2.10 + - ipaddress==1.0.22 + - jinja2==2.10.1 - lockfile==0.12.2 - - lxml==4.3.2 + - lxml==4.4.1 - markupsafe==1.1.1 - myproxyclient==2.1.0 - netcdf4==1.4.0 - nose==1.3.7 - - pbr==5.1.3 + - pbr==5.4.3 - pika==0.11.2 - - psutil==5.6.1 + - psutil==5.6.3 - psycopg2==2.6.2 - - regrid2==3.0.0 + - pycparser==2.19 + - pyopenssl==19.0.0 - requests==2.20.0 - - requests-cache==0.4.13 - - sqlalchemy==1.2.18 + - requests-cache==0.5.2 + - sqlalchemy==1.2.19 - sqlalchemy-migrate==0.11.0 - sqlparse==0.3.0 - tempita==0.5.2 - treelib==1.4.0 + - urllib3==1.24.3 From 3be8d5c9aeadbef530e9e132827ec5c360fc8762 Mon Sep 17 00:00:00 2001 From: Nathan Carlson Date: Fri, 4 Oct 2019 09:30:58 -0700 Subject: [PATCH 04/12] Ensure cdtime is present in the publisher env --- roles/publisher/files/environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/publisher/files/environment.yml b/roles/publisher/files/environment.yml index dafdc879..682dbcbe 100644 --- a/roles/publisher/files/environment.yml +++ b/roles/publisher/files/environment.yml @@ -7,6 +7,7 @@ dependencies: - blas=1.0 - bzip2=1.0.8 - ca-certificates=2019.8.28 + - cdtime - certifi=2019.9.11 - cftime=1.0.3.4 - cmor=3.5.0 From a31b2ccaaf37b7371a4ac8ccda5c1c62f7f593dc Mon Sep 17 00:00:00 2001 From: Nathan Carlson Date: Fri, 4 Oct 2019 10:31:18 -0700 Subject: [PATCH 05/12] Pin cdms package versions --- roles/publisher/files/environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/publisher/files/environment.yml b/roles/publisher/files/environment.yml index 682dbcbe..3e808a26 100644 --- a/roles/publisher/files/environment.yml +++ b/roles/publisher/files/environment.yml @@ -7,7 +7,7 @@ dependencies: - blas=1.0 - bzip2=1.0.8 - ca-certificates=2019.8.28 - - cdtime + - cdtime=3.1.2 - certifi=2019.9.11 - cftime=1.0.3.4 - cmor=3.5.0 @@ -19,6 +19,7 @@ dependencies: - jpeg=9c - json-c=0.13.1 - krb5=1.16.1 + - libcdms=3.1.2 - libcurl=7.65.3 - libedit=3.1.20181209 - libffi=3.2.1 From e6c24e62cdae2adb726def66a783b42367f6372a Mon Sep 17 00:00:00 2001 From: Nathan Carlson Date: Fri, 4 Oct 2019 10:40:09 -0700 Subject: [PATCH 06/12] Only install globus tools if user specifies globus credentials --- roles/base/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/base/tasks/main.yml b/roles/base/tasks/main.yml index dc2eaea5..67821a37 100644 --- a/roles/base/tasks/main.yml +++ b/roles/base/tasks/main.yml @@ -31,5 +31,6 @@ loop_var: base_task - name: Include Globus Tools + when: globus_user is defined and globus_pass is defined include_role: name: globus_tools \ No newline at end of file From 157f6c614d098593264ecfdae62dff6b07213802 Mon Sep 17 00:00:00 2001 From: Nathan Carlson Date: Fri, 4 Oct 2019 11:23:19 -0700 Subject: [PATCH 07/12] Continue to make corrections to esgf-pub env --- roles/publisher/files/environment.yml | 62 +++++++++++++++++++-------- 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/roles/publisher/files/environment.yml b/roles/publisher/files/environment.yml index 3e808a26..0b064be0 100644 --- a/roles/publisher/files/environment.yml +++ b/roles/publisher/files/environment.yml @@ -4,70 +4,96 @@ channels: - conda-forge dependencies: - _libgcc_mutex=0.1 - - blas=1.0 + - asn1crypto=0.24.0 + - attrs=19.1.0 - bzip2=1.0.8 - ca-certificates=2019.8.28 + - cdat_info=8.2 - cdtime=3.1.2 - certifi=2019.9.11 + - cffi=1.12.3 - cftime=1.0.3.4 + - chardet=3.0.4 - cmor=3.5.0 + - cryptography=2.7 - curl=7.65.3 + - decorator=4.4.0 + - distarray=2.12.2 + - enum34=1.1.6 + - esmf=7.1.0 + - esmpy=7.1.0 - expat=2.2.6 + - functools32=3.2.3.2 + - future=0.17.1 + - g2clib=1.6.0 - hdf4=4.2.13 - hdf5=1.10.5 - - intel-openmp=2019.4 + - ipaddress=1.0.22 + - ipython_genutils=0.2.0 + - jasper=1.900.1 - jpeg=9c - json-c=0.13.1 + - jsonschema=3.0.2 + - jupyter_core=4.5.0 - krb5=1.16.1 + - libblas=3.8.0 + - libcblas=3.8.0 - libcdms=3.1.2 + - libcf=1.0.3 - libcurl=7.65.3 + - libdrs=3.1.2 + - libdrs_f=3.1.2 - libedit=3.1.20181209 - libffi=3.2.1 - libgcc-ng=9.1.0 - libgfortran-ng=7.3.0 + - liblapack=3.8.0 - libnetcdf=4.6.2 + - libopenblas=0.3.6 + - libpng=1.6.37 - libssh2=1.8.2 - libstdcxx-ng=9.1.0 + - libtiff=4.0.10 - libuuid=2.32.1 - - mkl=2019.4 - - mkl-service=2.3.0 - - mkl_fft=1.0.14 - - mkl_random=1.1.0 + - mpi=1.0 + - mpich=3.2.1 + - nbformat=4.4.0 - ncurses=6.1 - - numpy=1.16.5 - - numpy-base=1.16.5 + - netcdf-fortran=4.4.5 + - numpy=1.16.4 - openssl=1.1.1d - pip=19.2.3 + - pycparser=2.19 + - pyopenssl=19.0.0 + - pyrsistent=0.15.4 + - pysocks=1.7.1 - python=2.7.16 - readline=7.0 - setuptools=41.2.0 - six=1.12.0 - sqlite=3.29.0 - tk=8.6.8 + - traitlets=4.3.2 - udunits2=2.2.25 + - urllib3=1.24.2 - wheel=0.33.6 + - xz=5.2.4 - zlib=1.2.11 + - zstd=1.3.7 - pip: - - asn1crypto==1.0.0 - cdf2cim==0.3.3.0 + - cdms2==3.0.0 - cf-python==3.0.1 - cfdm==1.7.8 - - cffi==1.12.3 - cfunits==3.2.2 - - chardet==3.0.4 - - cryptography==2.7 - - decorator==4.4.0 - - enum34==1.1.6 - esgcet==3.7.1 - esgconfigparser==0.1.17 - esgf-pyclient==0.2.1 - esgfpid==0.7.14 - esgprep==2.9.7 - - future==0.17.1 - fuzzywuzzy==0.16.0 - hurry-filesize==0.9 - idna==2.7 - - ipaddress==1.0.22 - jinja2==2.10.1 - lockfile==0.12.2 - lxml==4.4.1 @@ -79,8 +105,7 @@ dependencies: - pika==0.11.2 - psutil==5.6.3 - psycopg2==2.6.2 - - pycparser==2.19 - - pyopenssl==19.0.0 + - regrid2==3.0.0 - requests==2.20.0 - requests-cache==0.5.2 - sqlalchemy==1.2.19 @@ -88,4 +113,3 @@ dependencies: - sqlparse==0.3.0 - tempita==0.5.2 - treelib==1.4.0 - - urllib3==1.24.3 From 3f2670916b85a2ab8fee6434dc83d686ae80c745 Mon Sep 17 00:00:00 2001 From: Nathan Carlson Date: Fri, 4 Oct 2019 14:00:10 -0700 Subject: [PATCH 08/12] Use SHA-256 hash for tomcat user password --- group_vars/data.yml | 2 +- roles/tomcat/templates/server.xml.j2 | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/group_vars/data.yml b/group_vars/data.yml index 036bf89e..b0acbefd 100644 --- a/group_vars/data.yml +++ b/group_vars/data.yml @@ -3,7 +3,7 @@ thredds: content: "{{ esg.content }}/thredds" tomcat_user: name: "dnode_user" - pass: "{{ admin_pass |hash('md5') }}" + pass: "{{ admin_pass |password_hash('sha256') }}" roles: "tdrAdmin,tdsConfig" thredds_webapp: diff --git a/roles/tomcat/templates/server.xml.j2 b/roles/tomcat/templates/server.xml.j2 index 40b123c7..b05fa288 100644 --- a/roles/tomcat/templates/server.xml.j2 +++ b/roles/tomcat/templates/server.xml.j2 @@ -69,7 +69,7 @@ @@ -122,7 +122,7 @@ --> - +