Add changeprop

hieradata/hosts/changeprop151.yaml

@@ -0,0 +1,41 @@
+changeprop::jobqueue: true
+changeprop::high_traffic_jobs_config:
+  categoryMembershipChange:
+    concurrency: 200
+  # CNDPurge is quite low-volume, but it uses delayed execution,
+  # so avoid putting it together with other low-volume jobs so that it doesn't
+  # block execution for others.
+  cdnPurge:
+    concurrency: 5
+  # RecordLinks is normally low-volume, but could have big spikes
+  # when maintenance scripts are run. Elevated concurrency
+  RecordLintJob:
+    concurrency: 50
+    consumer_batch_size: 10
+  wikibase-addUsagesForPage:
+    concurrency: 5
+  # For cirrus search jobs the retries are built into the job itself,
+  # so disable the retries by change-prop. We need special rules for cirrus
+  # jobs because they need special configuration.
+  cirrusSearchCheckerJob:
+    disable_delayed_execution: true #T198462
+    retry_limit: 0
+    concurrency: 20
+  cirrusSearchDeleteArchive:
+    retry_limit: 0
+    concurrency: 5
+  cirrusSearchDeletePages:
+    retry_limit: 0
+    concurrency: 5
+  cirrusSearchIncomingLinkCount:
+    retry_limit: 0
+    concurrency: 15
+  cirrusSearchLinksUpdate:
+    retry_limit: 0
+    concurrency: 300
+  cirrusSearchLinksUpdatePrioritized:
+    retry_limit: 0
+    concurrency: 150
+  cirrusSearchOtherIndex:
+    retry_limit: 0
+    concurrency: 5

hieradata/hosts/mwtask171.yaml

@@ -33,3 +33,4 @@ nginx::keepalive_timeout: 2
 nginx::keepalive_requests: 150
 nginx::logrotate_number: 2
 nginx::use_graylog: true
+nginx::remove_apache: false

hieradata/hosts/mwtask181.yaml

@@ -33,3 +33,4 @@ nginx::keepalive_timeout: 2
 nginx::keepalive_requests: 150
 nginx::logrotate_number: 2
 nginx::use_graylog: true
+nginx::remove_apache: false

hieradata/hosts/test151.yaml

@@ -7,10 +7,12 @@ contactgroups: ['sre', 'mediawiki']
 
 jobrunner: true
 jobrunner::intensive: true
+
 mediawiki::jobqueue::runner::beta: true
 mediawiki::jobqueue::runner::cron: true
 mediawiki::jobqueue::runner::redis_ip: '10.0.15.118:6379'
 mediawiki::jobqueue::wiki: 'loginwikibeta'
+mediawiki::use_cpjobqueue: true
 
 role::mediawiki::use_strict_firewall: true
 
@@ -47,6 +49,7 @@ nginx::keepalive_timeout: 75
 nginx::keepalive_requests: 1000
 nginx::logrotate_number: 2
 nginx::use_graylog: true
+nginx::remove_apache: false
 
 # memcached
 role::memcached::threads: ~

manifests/site.pp

@@ -10,6 +10,12 @@
     include role::irc
 }
 
+node 'changeprop151.wikitide.net' {
+    include base
+    include role::changeprop
+    include role::redis
+}
+
 node /^cloud1[5678]\.wikitide\.net$/ {
     include base
     include role::cloud

modules/changeprop/files/default.config.yaml

@@ -0,0 +1,53 @@
+# Number of worker processes to spawn.
+# Set to 0 to run everything in a single process without clustering.
+# Use 'ncpu' to run as many workers as there are CPU units
+num_workers: 1
+
+# Log error messages and gracefully restart a worker if v8 reports that it
+# uses more heap (note: not RSS) than this many mb.
+worker_heap_limit_mb: 150
+
+# Logger info
+logging:
+  name: changeprop
+  level: info
+
+metrics:
+  type: log
+
+services:
+  - name: changeprop
+    # a relative path or the name of an npm package, if different from name
+    module: hyperswitch
+    # optionally, a version constraint of the npm package
+    # version: ^0.4.0
+    # per-service config
+    conf:
+      # the port to bind to
+      port: 7200
+      # IP address to bind to, all IPs by default
+      # interface: localhost # uncomment to only listen on localhost
+      # allow cross-domain requests to the API (default '*')
+      cors: '*'
+      # to disable use:
+      # cors: false
+      # to restrict to a particular domain, use:
+      # cors: restricted.domain.org
+      # URL of the outbound proxy to use (complete with protocol)
+      # proxy:
+      spec:
+        title: The Change Propagation root
+        paths:
+          /{domain:a}/sys/queue:
+            x-modules:
+              - path: sys/kafka.js
+                options:
+                  metadata_broker_list: 10.0.18.146:9092
+                  dc_name: default
+                  concurrency: 10
+                  templates: {}
+                  # Useful for mirroring the propagation of events in a test
+                  # environment; test_mode does the following: a) disables
+                  # production entirely to prevent enqueuing retries, b) alters
+                  # the consumer group ID, and c) disables offset commits.
+                  test_mode: false

modules/changeprop/manifests/init.pp

@@ -0,0 +1,76 @@
+# == Class: changeprop
+
+class changeprop (
+    $broker_list              = lookup('changeprop::broker_list', {'default_value' => '10.0.18.146:9092'}),
+    $jobrunner_host           = lookup('changeprop::jobrunner_host', {'default_value' => 'jobrunner.wikitide.net'}),
+    $realm                    = lookup('changeprop::realm', {'default_value' => 'production'}),
+    $proxy                    = lookup('changeprop::proxy', {'default_value' => ''}),
+    $high_traffic_jobs_config = lookup('changeprop::high_traffic_jobs_config', {'default_value' => {}}),
+    $low_traffic_concurrency  = lookup('changeprop::low_traffic_concurrency', {'default_value' => 50}),
+    $redis_host               = lookup('changeprop::low_traffic_concurrency', {'default_value' => 'localhost'}),
+    $redis_password           = lookup('passwords::redis::master')
+) {
+    stdlib::ensure_packages(['nodejs', 'libsasl2-dev'])
+
+    group { 'changeprop':
+        ensure => present,
+    }
+
+    user { 'changeprop':
+        ensure     => present,
+        gid        => 'changeprop',
+        shell      => '/bin/false',
+        home       => '/srv/changeprop',
+        managehome => false,
+        system     => true,
+    }
+
+    git::clone { 'changeprop':
+        ensure             => present,
+        directory          => '/srv/changeprop',
+        origin             => 'https://github.com/miraheze/changeprop-deploy',
+        branch             => 'master',
+        owner              => 'changeprop',
+        group              => 'changeprop',
+        mode               => '0755',
+        recurse_submodules => true,
+        require            => [
+          User['changeprop'],
+          Group['changeprop'],
+        ],
+    }
+
+    file { '/etc/changeprop':
+        ensure  => directory,
+    }
+
+    if lookup('changeprop::jobqueue', {'default_value' => false}) {
+        file { '/etc/changeprop/config.yaml':
+            ensure  => present,
+            content => template('changeprop/jobqueue.config.yaml.erb'),
+            require => File['/etc/changeprop'],
+            notify  => Service['changeprop'],
+        }
+    } else {
+        file { '/etc/changeprop/config.yaml':
+            ensure  => present,
+            source  => 'puppet:///modules/changeprop/default.config.yaml',
+            require => File['/etc/changeprop'],
+            notify  => Service['changeprop'],
+        }
+    }
+
+    systemd::service { 'changeprop':
+        ensure  => present,
+        content => systemd_template('changeprop'),
+        restart => true,
+        require => Git::Clone['changeprop'],
+    }
+
+    monitoring::services { 'changeprop':
+        check_command => 'tcp',
+        vars          => {
+            tcp_port  => '7200',
+        },
+    }
+}

modules/changeprop/templates/initscripts/changeprop.systemd.erb

@@ -0,0 +1,21 @@
+[Unit]
+Description=ChangeProp Service
+Documentation=https://wikitech.wikimedia.org/wiki/Changeprop
+Wants=local-fs.target network.target
+After=local-fs.target network.target
+
+[Service]
+Type=simple
+User=changeprop
+Group=changeprop
+SyslogIdentifier=changeprop
+WorkingDirectory=/srv/changeprop
+ExecStart=/usr/bin/node /srv/changeprop/src/server.js -c /etc/changeprop/config.yaml
+KillMode=process
+Restart=on-success
+Restart=always
+PrivateTmp=true
+StandardOutput=syslog
+
+[Install]
+WantedBy=multi-user.target