From c4f7a23c4fbd01f64b68a7c1ff4292dcf477078e Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 28 Jan 2016 13:20:33 -0500 Subject: [PATCH 1/9] Don't check for vagrant user, that doesn't work on non-Vagrant installs --- util/vagrant/migrate.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/util/vagrant/migrate.sh b/util/vagrant/migrate.sh index 6e63dcb2af6..735bcaf33d6 100755 --- a/util/vagrant/migrate.sh +++ b/util/vagrant/migrate.sh @@ -64,14 +64,8 @@ confirm_proceed () { } # Check we are in the right place, and have the info we need. - -if [[ "`whoami`" != "vagrant" ]]; then - echo "Run this from the vagrant account in your Open edX machine." - exit 1 -fi - if [[ ! -d /edx/app/edxapp ]]; then - echo "Run this from the vagrant account in your Open edX machine." + echo "Run this on your Open edX machine." exit 1 fi From 8bc98dba2331e57a584293a1c6a79f5b88dbe966 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 1 Feb 2016 13:50:20 -0500 Subject: [PATCH 2/9] Convert the 'restart certs' handler into a normal task --- playbooks/roles/certs/handlers/main.yml | 23 ----------------------- playbooks/roles/certs/tasks/deploy.yml | 17 +++++++---------- playbooks/roles/certs/tasks/main.yml | 7 ------- 3 files changed, 7 insertions(+), 40 deletions(-) delete mode 100644 playbooks/roles/certs/handlers/main.yml diff --git a/playbooks/roles/certs/handlers/main.yml b/playbooks/roles/certs/handlers/main.yml deleted file mode 100644 index 816d753fdf1..00000000000 --- a/playbooks/roles/certs/handlers/main.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/edx/configuration -# wiki: https://github.com/edx/configuration/wiki -# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT -# -# -# -# Handlers for role certs -# -# Overview: -# - -- name: restart certs - supervisorctl: > - name=certs - supervisorctl_path={{ supervisor_ctl }} - config={{ supervisor_cfg }} - state=restarted - when: certs_installed is defined and not disable_edx_services diff --git a/playbooks/roles/certs/tasks/deploy.yml b/playbooks/roles/certs/tasks/deploy.yml index dc1f189fc49..7cf47b3706a 100644 --- a/playbooks/roles/certs/tasks/deploy.yml +++ b/playbooks/roles/certs/tasks/deploy.yml @@ -5,14 +5,12 @@ src=certs.env.json.j2 dest={{ certs_app_dir }}/env.json sudo_user: "{{ certs_user }}" - notify: restart certs - name: create certificate auth file template: > src=certs.auth.json.j2 dest={{ certs_app_dir }}/auth.json sudo_user: "{{ certs_user }}" - notify: restart certs - name: writing supervisor script for certificates template: > @@ -27,14 +25,12 @@ state=link force=yes mode=0644 - notify: restart certs when: not disable_edx_services - name: create ssh script for git template: > src={{ certs_git_ssh|basename }}.j2 dest={{ certs_git_ssh }} owner={{ certs_user }} mode=750 - notify: restart certs # This key is only needed if you are pulling down a private # certificates repo @@ -43,7 +39,6 @@ content="{{ CERTS_GIT_IDENTITY }}" dest={{ certs_git_identity }} force=yes owner={{ certs_user }} mode=0600 when: CERTS_GIT_IDENTITY != "none" - notify: restart certs - name: checkout certificates repo into {{ certs_code_dir }} git: > @@ -54,7 +49,6 @@ GIT_SSH: "{{ certs_git_ssh }}" register: certs_checkout when: CERTS_GIT_IDENTITY != "none" - notify: restart certs - name: checkout certificates repo into {{ certs_code_dir }} git: > @@ -63,19 +57,16 @@ sudo_user: "{{ certs_user }}" register: certs_checkout when: CERTS_GIT_IDENTITY == "none" - notify: restart certs - name: remove read-only ssh key for the certs repo file: path={{ certs_git_identity }} state=absent when: CERTS_GIT_IDENTITY != "none" - notify: restart certs - name : install python requirements pip: > requirements="{{ certs_requirements_file }}" virtualenv="{{ certs_venv_dir }}" state=present extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}" sudo_user: "{{ certs_user }}" - notify: restart certs # call supervisorctl update. this reloads # the supervisorctl config and restarts @@ -110,4 +101,10 @@ - include: tag_ec2.yml tags=deploy when: COMMON_TAG_EC2_INSTANCE -- set_fact: certs_installed=true +- name: restart certs + supervisorctl: > + name=certs + supervisorctl_path={{ supervisor_ctl }} + config={{ supervisor_cfg }} + state=restarted + when: not disable_edx_services diff --git a/playbooks/roles/certs/tasks/main.yml b/playbooks/roles/certs/tasks/main.yml index d87b2d48ed1..51ad178fd24 100644 --- a/playbooks/roles/certs/tasks/main.yml +++ b/playbooks/roles/certs/tasks/main.yml @@ -37,7 +37,6 @@ home="{{ certs_app_dir }}" createhome=no shell=/bin/false - notify: restart certs - name: create certs app dirs file: > @@ -45,7 +44,6 @@ state=directory owner="{{ certs_user }}" group="{{ common_web_group }}" - notify: restart certs with_items: - "{{ certs_app_dir }}" # needed for the ansible 1.5 git module @@ -68,14 +66,12 @@ path="{{ certs_gpg_dir }}" state=directory owner="{{ common_web_user }}" mode=0700 - notify: restart certs - name: copy the private gpg signing key copy: > src={{ CERTS_LOCAL_PRIVATE_KEY }} dest={{ certs_app_dir }}/{{ CERTS_LOCAL_PRIVATE_KEY|basename }} owner={{ common_web_user }} mode=0600 - notify: restart certs register: certs_gpg_key - name: copy the pgp trust export @@ -83,20 +79,17 @@ content="{{ CERTS_OWNER_TRUST }}" dest={{ certs_app_dir }}/trust.export owner={{ common_web_user }} mode=0600 - notify: restart certs - name: load the gpg key shell: > /usr/bin/gpg --homedir {{ certs_gpg_dir }} --import {{ certs_app_dir }}/{{ CERTS_LOCAL_PRIVATE_KEY|basename }} sudo_user: "{{ common_web_user }}" when: certs_gpg_key.changed - notify: restart certs - name: import the trust export shell: > /usr/bin/gpg --homedir {{ certs_gpg_dir }} --import-ownertrust {{ certs_app_dir }}/trust.export sudo_user: "{{ common_web_user }}" when: certs_gpg_key.changed - notify: restart certs - include: deploy.yml tags=deploy From 70e779e4266246031570dacfa7c5a110008c86b7 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 1 Feb 2016 13:52:15 -0500 Subject: [PATCH 3/9] Convert the 'restart xqueue' handler into a normal task --- playbooks/roles/xqueue/handlers/main.yml | 10 --------- playbooks/roles/xqueue/tasks/deploy.yml | 27 +++++++++--------------- playbooks/roles/xqueue/tasks/main.yml | 6 ------ 3 files changed, 10 insertions(+), 33 deletions(-) delete mode 100644 playbooks/roles/xqueue/handlers/main.yml diff --git a/playbooks/roles/xqueue/handlers/main.yml b/playbooks/roles/xqueue/handlers/main.yml deleted file mode 100644 index 950d97594ad..00000000000 --- a/playbooks/roles/xqueue/handlers/main.yml +++ /dev/null @@ -1,10 +0,0 @@ -- name: restart xqueue - supervisorctl: > - name={{ item }} - supervisorctl_path={{ supervisor_ctl }} - config={{ supervisor_cfg }} - state=restarted - when: xqueue_installed is defined and not disable_edx_services - with_items: - - xqueue - - xqueue_consumer diff --git a/playbooks/roles/xqueue/tasks/deploy.yml b/playbooks/roles/xqueue/tasks/deploy.yml index c50cbf6d31c..8a646d56434 100644 --- a/playbooks/roles/xqueue/tasks/deploy.yml +++ b/playbooks/roles/xqueue/tasks/deploy.yml @@ -23,8 +23,6 @@ template: > src=xqueue_gunicorn.py.j2 dest={{ xqueue_app_dir }}/xqueue_gunicorn.py sudo_user: "{{ xqueue_user }}" - notify: - - restart xqueue tags: - install - install:configuration @@ -32,8 +30,6 @@ - name: create xqueue application config template: src=xqueue.env.json.j2 dest={{ xqueue_app_dir }}/xqueue.env.json mode=0644 sudo_user: "{{ xqueue_user }}" - notify: - - restart xqueue tags: - install - install:configuration @@ -41,8 +37,6 @@ - name: create xqueue auth file template: src=xqueue.auth.json.j2 dest={{ xqueue_app_dir }}/xqueue.auth.json mode=0644 sudo_user: "{{ xqueue_user }}" - notify: - - restart xqueue tags: - install - install:configuration @@ -54,8 +48,6 @@ accept_hostkey=yes sudo_user: "{{ xqueue_user }}" register: xqueue_checkout - notify: - - restart xqueue tags: - install - install:code @@ -67,8 +59,6 @@ requirements="{{ xqueue_pre_requirements_file }}" virtualenv="{{ xqueue_venv_dir }}" state=present extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w" sudo_user: "{{ xqueue_user }}" - notify: - - restart xqueue tags: - install - install:app-requirements @@ -79,8 +69,6 @@ requirements="{{ xqueue_post_requirements_file }}" virtualenv="{{ xqueue_venv_dir }}" state=present extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w" sudo_user: "{{ xqueue_user }}" - notify: - - restart xqueue tags: - install - install:app-requirements @@ -95,8 +83,6 @@ DB_MIGRATION_USER: "{{ COMMON_MYSQL_MIGRATE_USER }}" DB_MIGRATION_PASS: "{{ COMMON_MYSQL_MIGRATE_PASS }}" when: migrate_db is defined and migrate_db|lower == "yes" and COMMON_MYSQL_MIGRATE_PASS - notify: - - restart xqueue tags: - migrate - migrate:db @@ -105,8 +91,6 @@ shell: > SERVICE_VARIANT=xqueue {{ xqueue_venv_bin }}/django-admin.py update_users --settings=xqueue.aws_settings --pythonpath={{ xqueue_code_dir }} sudo_user: "{{ xqueue_user }}" - notify: - - restart xqueue tags: - manage - manage:app-users @@ -148,4 +132,13 @@ - install - install:app-requirements -- set_fact: xqueue_installed=true +- name: restart xqueue + supervisorctl: > + name={{ item }} + supervisorctl_path={{ supervisor_ctl }} + config={{ supervisor_cfg }} + state=restarted + when: not disable_edx_services + with_items: + - xqueue + - xqueue_consumer diff --git a/playbooks/roles/xqueue/tasks/main.yml b/playbooks/roles/xqueue/tasks/main.yml index f836b54e8e8..87ebd259039 100644 --- a/playbooks/roles/xqueue/tasks/main.yml +++ b/playbooks/roles/xqueue/tasks/main.yml @@ -12,8 +12,6 @@ home="{{ xqueue_app_dir }}" createhome=no shell=/bin/false - notify: - - restart xqueue tags: - install - install:base @@ -24,8 +22,6 @@ state=directory owner="{{ xqueue_user }}" group="{{ common_web_group }}" - notify: - - restart xqueue with_items: - "{{ xqueue_app_dir }}" - "{{ xqueue_venvs_dir }}" @@ -35,8 +31,6 @@ - name: install a bunch of system packages on which xqueue relies apt: pkg={{','.join(xqueue_debian_pkgs)}} state=present - notify: - - restart xqueue tags: - install - install:system-requirements From 19e0127825dff214b6d76cb840f979e369fc7695 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 5 Feb 2016 11:42:50 -0500 Subject: [PATCH 4/9] Add the forums migration for teams discussion filtering --- util/vagrant/migrate.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/util/vagrant/migrate.sh b/util/vagrant/migrate.sh index 735bcaf33d6..08a9e15acca 100755 --- a/util/vagrant/migrate.sh +++ b/util/vagrant/migrate.sh @@ -167,6 +167,22 @@ make_config_venv # Dogwood details if [[ $TARGET == *dogwood* ]] ; then + # Run the forum migrations. + cat > migrate-008-context.js <<"EOF" + // from: https://github.com/edx/cs_comments_service/blob/master/scripts/db/migrate-008-context.js + print ("Add the new indexes for the context field"); + db.contents.ensureIndex({ _type: 1, course_id: 1, context: 1, pinned: -1, created_at: -1 }, {background: true}) + db.contents.ensureIndex({ _type: 1, commentable_id: 1, context: 1, pinned: -1, created_at: -1 }, {background: true}) + + print ("Adding context to all comment threads where it does not yet exist\n"); + var bulk = db.contents.initializeUnorderedBulkOp(); + bulk.find( {_type: "CommentThread", context: {$exists: false}} ).update( {$set: {context: "course"}} ); + bulk.execute(); + printjson (db.runCommand({ getLastError: 1, w: "majority", wtimeout: 5000 } )); +EOF + + mongo cs_comments_service migrate-008-context.js + # We are upgrading Python from 2.7.3 to 2.7.10, so remake the venvs. sudo rm -rf /edx/app/*/v*envs/* @@ -218,6 +234,10 @@ if [[ $TARGET == *dogwood* ]] ; then /edx/app/xqueue/xqueue/manage.py migrate \ --settings=xqueue.aws_settings --noinput --fake-initial fi + + # Run the forums migrations again to catch things made while this script + # was running. + mongo cs_comments_service migrate-008-context.js fi cd configuration/playbooks From 41f4aa2524f9911ae6d2cc70af7eb2bc5412f500 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 8 Feb 2016 16:50:45 -0500 Subject: [PATCH 5/9] Add two more management commands, and make manage.py invocation uniform --- util/vagrant/migrate.sh | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/util/vagrant/migrate.sh b/util/vagrant/migrate.sh index 08a9e15acca..f4fdb190a51 100755 --- a/util/vagrant/migrate.sh +++ b/util/vagrant/migrate.sh @@ -84,6 +84,11 @@ if [[ $CONFIGURATION == none ]]; then exit 1 fi +APPUSER=edxapp +if [[ $CONFIGURATION == fullstack ]] ; then + APPUSER=www-data +fi + # Birch details if [[ $TARGET == *birch* && $INTERACTIVE == true ]] ; then @@ -204,7 +209,7 @@ EOF make_config_venv # Need to get rid of South from edx-platform, or things won't work. - sudo -u edxapp /edx/app/edxapp/venvs/edxapp/bin/pip uninstall -y South + sudo -u $APPUSER /edx/bin/pip.edxapp uninstall -y South echo "Upgrading to the beginning of Django 1.8" cd configuration/playbooks/vagrant @@ -221,10 +226,8 @@ EOF echo "Running the Django 1.8 faked migrations" for item in lms cms; do - sudo -u edxapp \ - /edx/app/edxapp/venvs/edxapp/bin/python \ - /edx/app/edxapp/edx-platform/manage.py $item migrate \ - --settings=aws --noinput --fake-initial + sudo -u $APPUSER -E /edx/bin/python.edxapp \ + /edx/bin/manage.edxapp $item migrate --settings=aws --noinput --fake-initial done if [[ $CONFIGURATION == fullstack ]] ; then @@ -235,6 +238,13 @@ EOF --settings=xqueue.aws_settings --noinput --fake-initial fi + echo "Running fix-up management commands" + sudo -u $APPUSER -E /edx/bin/python.edxapp \ + /edx/bin/manage.edxapp lms --settings=aws generate_course_overview --all + + sudo -u $APPUSER -E /edx/bin/python.edxapp \ + /edx/bin/manage.edxapp lms --settings=aws post_cohort_membership_fix --commit + # Run the forums migrations again to catch things made while this script # was running. mongo cs_comments_service migrate-008-context.js From 6e59c519f57f845281c14c076e0a384c86ccccc3 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 9 Feb 2016 11:38:47 -0500 Subject: [PATCH 6/9] Run the fixups after we get to the final version of the code. --- util/vagrant/migrate.sh | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/util/vagrant/migrate.sh b/util/vagrant/migrate.sh index f4fdb190a51..a2427d4e50e 100755 --- a/util/vagrant/migrate.sh +++ b/util/vagrant/migrate.sh @@ -237,19 +237,9 @@ EOF /edx/app/xqueue/xqueue/manage.py migrate \ --settings=xqueue.aws_settings --noinput --fake-initial fi - - echo "Running fix-up management commands" - sudo -u $APPUSER -E /edx/bin/python.edxapp \ - /edx/bin/manage.edxapp lms --settings=aws generate_course_overview --all - - sudo -u $APPUSER -E /edx/bin/python.edxapp \ - /edx/bin/manage.edxapp lms --settings=aws post_cohort_membership_fix --commit - - # Run the forums migrations again to catch things made while this script - # was running. - mongo cs_comments_service migrate-008-context.js fi +echo "Updating to final version of code" cd configuration/playbooks echo "edx_platform_version: $TARGET" > vars.yml echo "ora2_version: $TARGET" >> vars.yml @@ -262,6 +252,20 @@ sudo ansible-playbook \ --extra-vars="@vars.yml" \ $SERVER_VARS \ vagrant-$CONFIGURATION.yml +cd ../.. + +if [[ $TARGET == *dogwood* ]] ; then + echo "Running data fixup management commands" + sudo -u $APPUSER -E /edx/bin/python.edxapp \ + /edx/bin/manage.edxapp lms --settings=aws generate_course_overview --all + + sudo -u $APPUSER -E /edx/bin/python.edxapp \ + /edx/bin/manage.edxapp lms --settings=aws post_cohort_membership_fix --commit + + # Run the forums migrations again to catch things made while this script + # was running. + mongo cs_comments_service migrate-008-context.js +fi cd / sudo rm -rf $TEMPDIR From 72a5d963fb2d5e59a4a84c48ff380d4c2c762bc2 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 9 Feb 2016 14:57:48 -0500 Subject: [PATCH 7/9] In both devstack and fullstack, pip needs to run as edxapp --- util/vagrant/migrate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/vagrant/migrate.sh b/util/vagrant/migrate.sh index a2427d4e50e..cdadccccc48 100755 --- a/util/vagrant/migrate.sh +++ b/util/vagrant/migrate.sh @@ -209,7 +209,7 @@ EOF make_config_venv # Need to get rid of South from edx-platform, or things won't work. - sudo -u $APPUSER /edx/bin/pip.edxapp uninstall -y South + sudo -u edxapp /edx/bin/pip.edxapp uninstall -y South echo "Upgrading to the beginning of Django 1.8" cd configuration/playbooks/vagrant From 9a024a44184f5cfae0a91e161b02df94b0d58a52 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 10 Feb 2016 14:42:04 -0500 Subject: [PATCH 8/9] Dogwood final box names --- vagrant/release/devstack/Vagrantfile | 5 +---- vagrant/release/fullstack/Vagrantfile | 7 ++----- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/vagrant/release/devstack/Vagrantfile b/vagrant/release/devstack/Vagrantfile index ce08108db19..50c843e6e9b 100644 --- a/vagrant/release/devstack/Vagrantfile +++ b/vagrant/release/devstack/Vagrantfile @@ -63,10 +63,7 @@ openedx_releases = { "named-release/dogwood.rc" => { :name => "dogwood-devstack-rc2", :file => "20151221-dogwood-devstack-rc2.box", }, - "named-release/dogwood.rc2" => { - :name => "dogwood-devstack-rc2", :file => "20151221-dogwood-devstack-rc2.box", - }, - "named-release/dogwood.rc3" => { + "named-release/dogwood" => { :name => "dogwood-devstack-rc2", :file => "20151221-dogwood-devstack-rc2.box", }, "named-release/cypress" => { diff --git a/vagrant/release/fullstack/Vagrantfile b/vagrant/release/fullstack/Vagrantfile index 4ee0ba2762f..3c92168fbde 100644 --- a/vagrant/release/fullstack/Vagrantfile +++ b/vagrant/release/fullstack/Vagrantfile @@ -9,13 +9,10 @@ CPU_COUNT = 2 # to a name and a file path, which are used for retrieving # a Vagrant box from the internet. openedx_releases = { - "named-release/dogwood.rc" => { - :name => "dogwood-fullstack-rc2", :file => "20151221-dogwood-fullstack-rc2.box", - }, - "named-release/dogwood.rc2" => { + "named-release/dogwood" => { :name => "dogwood-fullstack-rc2", :file => "20151221-dogwood-fullstack-rc2.box", }, - "named-release/dogwood.rc3" => { + "named-release/dogwood.rc" => { :name => "dogwood-fullstack-rc2", :file => "20151221-dogwood-fullstack-rc2.box", }, "named-release/cypress" => { From 5a80d2e442fb9ca28fce88c8a4ac5ddc6617f726 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 10 Feb 2016 16:34:27 -0500 Subject: [PATCH 9/9] Remove the Cypress box names --- vagrant/release/devstack/Vagrantfile | 7 ++++--- vagrant/release/fullstack/Vagrantfile | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/vagrant/release/devstack/Vagrantfile b/vagrant/release/devstack/Vagrantfile index 50c843e6e9b..cc99e44aa07 100644 --- a/vagrant/release/devstack/Vagrantfile +++ b/vagrant/release/devstack/Vagrantfile @@ -66,9 +66,10 @@ openedx_releases = { "named-release/dogwood" => { :name => "dogwood-devstack-rc2", :file => "20151221-dogwood-devstack-rc2.box", }, - "named-release/cypress" => { - :name => "cypress-devstack", :file => "cypress-devstack.box", - }, + # Cypress is deprecated and unsupported + # "named-release/cypress" => { + # :name => "cypress-devstack", :file => "cypress-devstack.box", + # }, # Birch is deprecated and unsupported # "named-release/birch.2" => { # :name => "birch-devstack-2", :file => "birch-2-devstack.box", diff --git a/vagrant/release/fullstack/Vagrantfile b/vagrant/release/fullstack/Vagrantfile index 3c92168fbde..a6caa68ecef 100644 --- a/vagrant/release/fullstack/Vagrantfile +++ b/vagrant/release/fullstack/Vagrantfile @@ -15,9 +15,10 @@ openedx_releases = { "named-release/dogwood.rc" => { :name => "dogwood-fullstack-rc2", :file => "20151221-dogwood-fullstack-rc2.box", }, - "named-release/cypress" => { - :name => "cypress-fullstack", :file => "cypress-fullstack.box", - }, + # Cypress is deprecated and unsupported + # "named-release/cypress" => { + # :name => "cypress-fullstack", :file => "cypress-fullstack.box", + # }, # Birch is deprecated and unsupported # "named-release/birch.2" => { # :name => "birch-fullstack-2", :file => "birch-2-fullstack.box",