github ci: update image for testing

- image based on production v1.5.0 - introduce rdkit and openbabel gems compatible with ubuntu 22.04 - also fix datacollector initializer issue introduced with fc624fc - spec datacollector: sftp setting upd - set .tool-versions - skip capybara test for now - update-rb-node-version in config/docker files
ComPlat · Mar 20, 2023 · d590fc3 · d590fc3
1 parent 933c126
commit d590fc3
Show file tree

Hide file tree

Showing 13 changed files with 86 additions and 81 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -21,7 +21,7 @@ jobs:
 
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: '2.7.6'
+          ruby-version: '2.7.7'
 
       - name: install linting dependencies
         # IMPORTANT: install rubocop first for pinning to take effect (`gem` doesn't resolve dependencies but simply installs gems in order)
@@ -49,7 +49,7 @@ jobs:
       run:
         shell: bash
 
-    container: complat/chemotion_eln_runner:ruby-2.7.6
+    container: complat/chemotion_eln_runner:v1.5.0
 
     services:
       postgres:
@@ -67,37 +67,24 @@ jobs:
       - name: checkout repository
         uses: actions/checkout@v3
 
-      - name: update package repository
-        run: sudo apt update
-
-      - name: install SSH service
-        run: sudo apt install -y openssh-server
-
-      - name: restart SSH server
-        run: sudo service ssh restart
-
-      - name: check SSH service
-        run: sudo service ssh status
-
-      - name: generate SSH key
-        run: |
-          mkdir -p $HOME/.ssh
-          ssh-keygen -t rsa -b 4096 -C "CI Deploy Key" -f $HOME/.ssh/id_rsa -N ""
-          echo "$(<$HOME/.ssh/id_rsa.pub)" >> $HOME/.ssh/authorized_keys
-
-      - name: create user
+      - name: create user for datacollectors testing
         run: |
-          sudo useradd testuser -m -s /bin/bash
-          sudo mkdir /home/testuser/.ssh
-          sudo chown testuser:testuser /home/testuser/.ssh
-          sudo chmod 700 /home/testuser/.ssh
+          useradd testuser -m -s /bin/bash
+          mkdir /home/testuser/.ssh
+          chown testuser:testuser /home/testuser/.ssh
+          chmod 700 /home/testuser/.ssh
 
-      - name: add SSH key to authorized_keys
+      - name: configure ssh for datacollectors testing 
         run: |
-          echo "$(<$HOME/.ssh/id_rsa.pub)" | sudo tee -a /home/testuser/.ssh/authorized_keys
-          cat /home/testuser/.ssh/authorized_keys
-          sudo chown testuser:testuser /home/testuser/.ssh/authorized_keys
-          sudo chmod 600 /home/testuser/.ssh/authorized_keys
+          service ssh restart
+          mkdir -p $HOME/.ssh
+          chmod 700 $HOME/.ssh
+          ssh-keygen -t ed25519 -f $HOME/.ssh/id_test -N ""
+          # echo "$(<$HOME/.ssh/id_test.pub)" >> $HOME/.ssh/authorized_keys
+          echo "$(<$HOME/.ssh/id_test.pub)" | sudo tee -a /home/testuser/.ssh/authorized_keys
+          eval `ssh-agent`
+          ssh-add $HOME/.ssh/id_test
+          ssh-keyscan -H 127.0.0.1  >> $HOME/.ssh/known_hosts
 
       - name: configure repository
         run: |
@@ -110,6 +97,7 @@ jobs:
           cp -f storage.yml.example storage.yml
           cp -f radar.yml.example radar.yml
           touch klasses.json
+          bundle config set without ''
 
       - name: configure postgres
         env:
@@ -126,26 +114,22 @@ jobs:
         run: bundle install
 
       - name: yarn install
-        run: yarn install
+        run:  yarn install --production=false
 
       - name: prepare postgres
         run: |
           RAILS_ENV=test bundle exec rake db:migrate
           RAILS_ENV=test bundle exec rake db:test:prepare
           RAILS_ENV=test bundle exec rake db:seed
 
-      - name: precompile
-        run: RAILS_ENV=test bundle exec rake webpacker:compile
-
       - name: npm test
         run: npm test
 
-      - name: rspec acceptance
-        continue-on-error: true    # don't fail job because this step; TODO: fix flaky acceptance tests...
-        run: RAILS_ENV=test bundle exec rspec spec/features
-
       - name: rspec unit
-        run: RAILS_ENV=test bundle exec rspec --exclude-pattern spec/{features}/**/*_spec.rb
+        run: |
+          eval `ssh-agent`
+          ssh-add $HOME/.ssh/id_test
+          RAILS_ENV=test bundle exec rspec --exclude-pattern spec/{features}/**/*_spec.rb spec
 
       - name: coverage rspec unit
         if: github.event.pull_request && (github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name) && (github.triggering_actor != 'dependabot[bot]')  # don't run on PRs from forks because of missing write permission: https://github.com/orgs/community/discussions/26829
@@ -155,3 +139,11 @@ jobs:
           minimum-coverage: 57
           artifact-name: code-coverage-report
           github-token: ${{ secrets.GITHUB_TOKEN }}
+
+#      - name: precompile
+#        run: RAILS_ENV=test bundle exec rake webpacker:compile
+
+#      - name: rspec acceptance
+#        continue-on-error: true    # don't fail job because this step; TODO: fix flaky acceptance tests...
+#        run: RAILS_ENV=test bundle exec rspec spec/features
+
diff --git a/.nvmrc b/.nvmrc
@@ -1 +1 @@
-v14.20.0
+v14.21.3
diff --git a/.ruby-version.example b/.ruby-version.example
@@ -1 +1 @@
-2.7.6
+2.7.7
diff --git a/.tool-versions b/.tool-versions
@@ -0,0 +1,2 @@
+nodejs 14.21.3
+ruby 2.7.7
diff --git a/Dockerfile.chemotion-dev b/Dockerfile.chemotion-dev
@@ -6,10 +6,10 @@
 FROM --platform=linux/amd64 ubuntu:focal
 
 ARG DEBIAN_FRONTEND=noninteractive
-ARG VRUBY=2.7.6
-ARG VNODE=14.20.0
+ARG VRUBY=2.7.7
+ARG VNODE=14.21.3
 ARG VNODENEXT=16.16.0
-ARG ASDF_BRANCH=v0.8.1
+ARG ASDF_BRANCH=v0.11.3
 
 RUN set -xe  && apt-get update -yqqq --fix-missing && apt-get upgrade -y
 RUN apt update && apt-get install -yqq --fix-missing bash ca-certificates wget apt-transport-https git gpg\

diff --git a/Dockerfile.focal.gitlab-ci b/Dockerfile.focal.gitlab-ci
@@ -1,8 +1,8 @@
 FROM ubuntu:focal
 
 ARG DEBIAN_FRONTEND=noninteractive
-ARG VRUBY=2.7.6
-ARG VNODE=14.20.0
+ARG VRUBY=2.7.7
+ARG VNODE=14.21.3
 ARG VNODENEXT=16.16.0
 
 RUN set -xe  && apt-get update -yqqq --fix-missing && apt-get upgrade -y

diff --git a/Dockerfile.github-ci b/Dockerfile.github-ci
@@ -1,6 +1,6 @@
-FROM ruby:2.7.6
+FROM ruby:2.7.7
 
-ARG NODE_VERSION=14.20.0
+ARG NODE_VERSION=14.21.3
 ARG BUNDLER_VERSION=2.1.4
 
 RUN apt-get update && apt-get upgrade -y && apt-get install -y \

diff --git a/Gemfile b/Gemfile
@@ -4,6 +4,7 @@ source 'https://rubygems.org'
 
 gem 'aasm'
 gem 'activejob-status'
+gem 'activerecord-nulldb-adapter'
 gem 'ancestry'
 gem 'api-pagination'
 gem 'caxlsx'
@@ -73,8 +74,7 @@ gem 'omniauth-github', '~> 1.4.0'
 gem 'omniauth-oauth2', '~> 1.7', '>= 1.7.2'
 gem 'omniauth_openid_connect'
 gem 'omniauth-orcid', git: 'https://github.com/datacite/omniauth-orcid'
-
-gem 'openbabel', '2.4.90.3', git: 'https://github.com/ComPlat/openbabel-gem.git', branch: 'hot-fix-svg'
+gem 'openbabel', '2.4.90.3', git: 'https://github.com/ptrxyz/openbabel-gem.git', branch: 'ptrxyz-ctime-fix'
 
 gem 'pandoc-ruby'
 gem 'paranoia'
@@ -88,7 +88,7 @@ gem 'pundit'
 gem 'rack'
 gem 'rack-cors', require: 'rack/cors'
 gem 'rails', '6.1.7'
-gem 'rdkit_chem', git: 'https://github.com/CamAnNguyen/rdkit_chem'
+gem 'rdkit_chem', git: 'https://github.com/ptrxyz/rdkit_chem.git', branch: 'pk01'
 gem 'rinchi-gem', '1.0.1', git: 'https://git.scc.kit.edu/ComPlat/rinchi-gem.git'
 gem 'rmagick'
 gem 'roo'
@@ -111,6 +111,7 @@ gem 'sys-filesystem'
 gem 'thor'
 gem 'thumbnailer', git: 'https://github.com/merlin-p/thumbnailer.git'
 gem 'turbo-sprockets-rails4'
+gem 'tzinfo-data'
 
 gem 'webpacker', git: 'https://github.com/rails/webpacker', branch: 'master'
 gem 'whenever', require: false

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -4,26 +4,13 @@ GIT
   specs:
     rinchi-gem (1.0.1)
 
-GIT
-  remote: https://github.com/CamAnNguyen/rdkit_chem
-  revision: cf8a3890c5aa7f4db74e40d1cbd3a50b62bd8d94
-  specs:
-    rdkit_chem (2020.04.30.1)
-
 GIT
   remote: https://github.com/ComPlat/inchi-gem.git
   revision: 9a95669b42e18a20d2a0c2d5440670ff94e46c37
   branch: main
   specs:
     inchi-gem (1.06.1)
 
-GIT
-  remote: https://github.com/ComPlat/openbabel-gem.git
-  revision: 3e25548fd95c2ac83c7fb5d9544ed2cf6ae1344b
-  branch: hot-fix-svg
-  specs:
-    openbabel (2.4.90.3)
-
 GIT
   remote: https://github.com/ComPlat/sablon
   revision: 0bb0e77a8eb5ed2953d81af42558e568845c3777
@@ -92,6 +79,20 @@ GIT
   specs:
     thumbnailer (0.2.0)
 
+GIT
+  remote: https://github.com/ptrxyz/openbabel-gem.git
+  revision: a1255a58c7fa52a00b5743dce46af714365d6acd
+  branch: ptrxyz-ctime-fix
+  specs:
+    openbabel (2.4.90.3)
+
+GIT
+  remote: https://github.com/ptrxyz/rdkit_chem.git
+  revision: c8ddea813688df7014606a1c5802b25b1fcbe3f6
+  branch: pk01
+  specs:
+    rdkit_chem (2020.04.30.1)
+
 GIT
   remote: https://github.com/rails/webpacker
   revision: 032c2d11a3758035305fe34e6795ab133128ee32
@@ -168,6 +169,8 @@ GEM
     activerecord (6.1.7)
       activemodel (= 6.1.7)
       activesupport (= 6.1.7)
+    activerecord-nulldb-adapter (0.9.0)
+      activerecord (>= 5.2.0, < 7.1)
     activestorage (6.1.7)
       actionpack (= 6.1.7)
       activejob (= 6.1.7)
@@ -779,6 +782,8 @@ GEM
       sprockets (~> 3.0)
     tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2022.7)
+      tzinfo (>= 1.0.0)
     unicode-display_width (2.3.0)
     uniform_notifier (1.16.0)
     validate_email (0.1.6)
@@ -830,6 +835,7 @@ PLATFORMS
 DEPENDENCIES
   aasm
   activejob-status
+  activerecord-nulldb-adapter
   ancestry
   annotate
   api-pagination
@@ -956,6 +962,7 @@ DEPENDENCIES
   thor
   thumbnailer!
   turbo-sprockets-rails4
+  tzinfo-data
   web-console
   webdrivers
   webmock

diff --git a/config/initializers/datacollectors.rb b/config/initializers/datacollectors.rb
@@ -5,12 +5,14 @@
 
 if File.file?(Rails.root.join('config/datacollectors.yml'))
   datacollectors_config = Rails.application.config_for :datacollectors
-  Rails.application.configure do
-    config.datacollectors = ActiveSupport::OrderedOptions.new
-    config.datacollectors.services = datacollectors_config[:services]
-    config.datacollectors.mailcollector = datacollectors_config[:mailcollector]
-    config.datacollectors.sftpusers = datacollectors_config[:sftpusers]
-    config.datacollectors.localcollectors = datacollectors_config[:localcollectors]
-    config.datacollectors.keydir = datacollectors_config[:keydir]
+  if datacollectors_config
+    Rails.application.configure do
+      config.datacollectors = ActiveSupport::OrderedOptions.new
+      config.datacollectors.services = datacollectors_config[:services]
+      config.datacollectors.mailcollector = datacollectors_config[:mailcollector]
+      config.datacollectors.sftpusers = datacollectors_config[:sftpusers]
+      config.datacollectors.localcollectors = datacollectors_config[:localcollectors]
+      config.datacollectors.keydir = datacollectors_config[:keydir]
+    end
   end
 end
diff --git a/lib/datacollector/fcollector.rb b/lib/datacollector/fcollector.rb
@@ -20,7 +20,6 @@ def execute(use_sftp)
       @current_collector = nil
       method_params = device.profile.data['method_params']
       host = method_params['host']
-
       case method_params['authen']
       when 'keyfile'
         user = method_params['user']
@@ -33,6 +32,8 @@ def execute(use_sftp)
         args = {
           key_data: [],
           keys: kp,
+          auth_methods: %w[publickey],
+          verbose: :error,
           keys_only: true,
         }
       when 'password', nil

diff --git a/spec/factories/devices.rb b/spec/factories/devices.rb
@@ -36,10 +36,10 @@
             'method' => 'filewatchersftp',
             'method_params' => {
               'dir' => "#{Rails.root}/tmp/datacollector/#{name_abbreviation}",
-              'user' => 'testuser',
+              'user' => ENV['DATACOLLECTOR_TEST_USER'].presence || 'testuser',
               'host' => '127.0.0.1',
               'authen' => 'keyfile',
-              'key_name' => "#{Dir.home}/.ssh/id_rsa",
+              'key_name' => "#{Dir.home}/.ssh/id_test",
               'number_of_files' => 1,
             },
           },
@@ -57,7 +57,7 @@
               'user' => 'dummy',
               'host' => '127.0.0.1',
               'authen' => 'keyfile',
-              'key_name' => "#{Dir.home}/.ssh/id_rsa",
+              'key_name' => "#{Dir.home}/.ssh/id_test",
               'number_of_files' => 1,
             },
           },
@@ -87,10 +87,10 @@
             'method' => 'folderwatchersftp',
             'method_params' => {
               'dir' => "#{Rails.root}/tmp/datacollector/#{name_abbreviation}",
-              'user' => 'testuser',
+              'user' => ENV['DATACOLLECTOR_TEST_USER'].presence || 'testuser',
               'host' => '127.0.0.1',
               'authen' => 'keyfile',
-              'key_name' => "#{Dir.home}/.ssh/id_rsa",
+              'key_name' => "#{Dir.home}/.ssh/id_test",
               'number_of_files' => 1,
             },
           },
@@ -108,7 +108,7 @@
               'user' => 'dummy',
               'host' => '127.0.0.1',
               'authen' => 'keyfile',
-              'key_name' => "#{Dir.home}/.ssh/id_rsa",
+              'key_name' => "#{Dir.home}/.ssh/id_test",
               'number_of_files' => 1,
             },
           },

diff --git a/spec/lib/datacollector/filecollector_spec.rb b/spec/lib/datacollector/filecollector_spec.rb
@@ -16,7 +16,7 @@
         device1
         device2
 
-        expect { described_class.new.execute(false) }.to change(Attachment, :count).by(Device.count)
+        expect { described_class.new.execute(false) }.to change(Attachment, :count).by(2)
       end
     end
 
@@ -25,7 +25,7 @@
         device_sftp1
         device_sftp3
 
-        expect { described_class.new.execute(true) }.to change(Attachment, :count).by(Device.count)
+        expect { described_class.new.execute(true) }.to change(Attachment, :count).by(2)
       end
     end
 
@@ -34,7 +34,7 @@
         device_sftp1
         device_sftp3
 
-        expect { described_class.new.execute(true) }.to change(Attachment, :count).by(Device.count)
+        expect { described_class.new.execute(true) }.to change(Attachment, :count).by(2)
       end
     end