From 143108768189a2c41735617815f4c4d09c5b93a5 Mon Sep 17 00:00:00 2001 From: petergmurphy Date: Wed, 15 Jan 2025 15:12:11 +0000 Subject: [PATCH 1/4] (PE-39577) Optimise legacy compiler support This commit: - Adds the `node_group_unpin` task. - `node_group_unpin` task is called in the convert plan to remove legacy compilers from from the PE Master node group. - Legacy compilers `pp_auth_role` changed to `legacy_compiler`. - Changes the PEADM config to use the PE Certificate Authority node group. - Removes peadm_legacy_compiler extension. --- REFERENCE.md | 21 ++++ manifests/setup/legacy_compiler_group.pp | 59 ++++++------ manifests/setup/node_manager.pp | 19 ++-- plans/convert.pp | 18 ++-- plans/convert_compiler_to_legacy.pp | 8 +- plans/subplans/component_install.pp | 4 +- plans/subplans/install.pp | 8 +- plans/update_compiler_extensions.pp | 5 - plans/upgrade.pp | 8 +- spec/plans/convert_spec.rb | 4 +- tasks/get_peadm_config.rb | 8 +- tasks/node_group_unpin.json | 17 ++++ tasks/node_group_unpin.rb | 117 +++++++++++++++++++++++ 13 files changed, 220 insertions(+), 76 deletions(-) create mode 100644 tasks/node_group_unpin.json create mode 100755 tasks/node_group_unpin.rb diff --git a/REFERENCE.md b/REFERENCE.md index e7563f9c..3fc16531 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -74,6 +74,7 @@ * [`infrastatus`](#infrastatus): Runs puppet infra status and returns the output * [`mkdir_p_file`](#mkdir_p_file): Create a file with the specified content at the specified location * [`mv`](#mv): Wrapper task for mv command +* [`node_group_unpin`](#node_group_unpin): Unpins a node from a specified PE node group * [`os_identification`](#os_identification): Return the operating system runnin gon the target as a string * [`pe_install`](#pe_install): Install Puppet Enterprise from a tarball * [`pe_ldap_config`](#pe_ldap_config): Set the ldap config in the PE console @@ -1324,6 +1325,26 @@ Data type: `String` New path of file +### `node_group_unpin` + +Unpins a node from a specified PE node group + +**Supports noop?** false + +#### Parameters + +##### `node_certname` + +Data type: `String` + +The certname of the node to unpin + +##### `group_name` + +Data type: `String` + +The name of the node group to unpin the node from + ### `os_identification` Return the operating system runnin gon the target as a string diff --git a/manifests/setup/legacy_compiler_group.pp b/manifests/setup/legacy_compiler_group.pp index e3601200..8c31a9c0 100644 --- a/manifests/setup/legacy_compiler_group.pp +++ b/manifests/setup/legacy_compiler_group.pp @@ -1,43 +1,43 @@ # @api private class peadm::setup::legacy_compiler_group ( String[1] $primary_host, - Optional[String] $internal_compiler_a_pool_address = undef, - Optional[String] $internal_compiler_b_pool_address = undef, + Optional[String] $internal_compiler_a_pool_address = undef, + Optional[String] $internal_compiler_b_pool_address = undef, ) { Node_group { purge_behavior => none, } node_group { 'PE Legacy Compiler': - parent => 'PE Master', - rule => ['and', - ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'true'], - ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], - ], - classes => { - 'puppet_enterprise::profile::master' => { - 'puppetdb_host' => [$internal_compiler_a_pool_address, $internal_compiler_b_pool_address].filter |$_| { $_ }, - 'puppetdb_port' => [8081], + ensure => 'present', + parent => 'PE Master', + purge_behavior => 'classes', + rule => ['=', ['trusted', 'extensions', 'pp_auth_role'], 'legacy_compiler'], + classes => { + 'puppet_enterprise::profile::master' => { + 'puppetdb_host' => [$internal_compiler_a_pool_address, $internal_compiler_b_pool_address].filter |$_| { $_ }, + 'puppetdb_port' => [8081], + 'replication_mode' => 'none', + 'code_manager_auto_configure' => true, }, }, } node_group { 'PE Legacy Compiler Group A': - ensure => 'present', - parent => 'PE Legacy Compiler', - rule => ['and', - ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], + ensure => 'present', + parent => 'PE Legacy Compiler', + purge_behavior => 'classes', + rule => ['and', + ['=', ['trusted', 'extensions', 'pp_auth_role'], 'legacy_compiler'], ['=', ['trusted', 'extensions', peadm::oid('peadm_availability_group')], 'A'], - ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'true'], ], - classes => { - 'puppet_enterprise::profile::master' => { + classes => { + 'puppet_enterprise::profile::master' => { 'puppetdb_host' => [$internal_compiler_b_pool_address, $internal_compiler_a_pool_address].filter |$_| { $_ }, 'puppetdb_port' => [8081], }, }, - data => { - # Workaround for GH-118 + data => { 'puppet_enterprise::profile::master::puppetdb' => { 'ha_enabled_replicas' => [], }, @@ -45,21 +45,20 @@ } node_group { 'PE Legacy Compiler Group B': - ensure => 'present', - parent => 'PE Legacy Compiler', - rule => ['and', - ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], + ensure => 'present', + parent => 'PE Legacy Compiler', + purge_behavior => 'classes', + rule => ['and', + ['=', ['trusted', 'extensions', 'pp_auth_role'], 'legacy_compiler'], ['=', ['trusted', 'extensions', peadm::oid('peadm_availability_group')], 'B'], - ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'true'], ], - classes => { - 'puppet_enterprise::profile::master' => { + classes => { + 'puppet_enterprise::profile::master' => { 'puppetdb_host' => [$internal_compiler_a_pool_address, $internal_compiler_b_pool_address].filter |$_| { $_ }, 'puppetdb_port' => [8081], }, }, - data => { - # Workaround for GH-118 + data => { 'puppet_enterprise::profile::master::puppetdb' => { 'ha_enabled_replicas' => [], }, @@ -67,6 +66,6 @@ } node_group { 'PE Compiler': - rule => ['and', ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'false']], + rule => ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], } } diff --git a/manifests/setup/node_manager.pp b/manifests/setup/node_manager.pp index f74cb217..5334a3f1 100644 --- a/manifests/setup/node_manager.pp +++ b/manifests/setup/node_manager.pp @@ -77,12 +77,16 @@ parent => 'PE Infrastructure', data => $compiler_pool_address_data, variables => { 'pe_master' => true }, + rule => ['or', + ['=', ['trusted', 'extensions', 'pp_auth_role'], 'legacy_compiler'], + ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], + ], } # PE Compiler group comes from default PE and already has the pe compiler role node_group { 'PE Compiler': parent => 'PE Master', - rule => ['and', ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'false']], + rule => ['and', ['=', ['trusted', 'extensions', peadm::oid('pp_auth_role')], 'pe_compiler']], } # This group should pin the primary, and also map to any pe-postgresql nodes @@ -121,7 +125,6 @@ rule => ['and', ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], ['=', ['trusted', 'extensions', peadm::oid('peadm_availability_group')], 'A'], - ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'false'], ], classes => { 'puppet_enterprise::profile::puppetdb' => { @@ -180,7 +183,6 @@ rule => ['and', ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], ['=', ['trusted', 'extensions', peadm::oid('peadm_availability_group')], 'B'], - ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'false'], ], classes => { 'puppet_enterprise::profile::puppetdb' => { @@ -203,10 +205,7 @@ node_group { 'PE Legacy Compiler': parent => 'PE Master', - rule => ['and', - ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], - ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'true'], - ], + rule => ['=', ['trusted', 'extensions', 'pp_auth_role'], 'legacy_compiler'], classes => { 'puppet_enterprise::profile::master' => { 'puppetdb_host' => [$internal_compiler_a_pool_address, $internal_compiler_b_pool_address].filter |$_| { $_ }, @@ -221,9 +220,8 @@ ensure => 'present', parent => 'PE Legacy Compiler', rule => ['and', - ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], + ['=', ['trusted', 'extensions', 'pp_auth_role'], 'legacy_compiler'], ['=', ['trusted', 'extensions', peadm::oid('peadm_availability_group')], 'A'], - ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'true'], ], classes => { 'puppet_enterprise::profile::master' => { @@ -245,9 +243,8 @@ ensure => 'present', parent => 'PE Legacy Compiler', rule => ['and', - ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], + ['=', ['trusted', 'extensions', 'pp_auth_role'], 'legacy_compiler'], ['=', ['trusted', 'extensions', peadm::oid('peadm_availability_group')], 'B'], - ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'true'], ], classes => { 'puppet_enterprise::profile::master' => { diff --git a/plans/convert.pp b/plans/convert.pp index 1ff1771a..11ee1209 100644 --- a/plans/convert.pp +++ b/plans/convert.pp @@ -214,7 +214,6 @@ add_extensions => { peadm::oid('pp_auth_role') => 'pe_compiler', peadm::oid('peadm_availability_group') => 'A', - peadm::oid('peadm_legacy_compiler') => 'false', }, ) }, @@ -224,7 +223,6 @@ add_extensions => { peadm::oid('pp_auth_role') => 'pe_compiler', peadm::oid('peadm_availability_group') => 'B', - peadm::oid('peadm_legacy_compiler') => 'false', }, ) }, @@ -232,9 +230,8 @@ run_plan('peadm::modify_certificate', $legacy_compiler_a_targets, primary_host => $primary_target, add_extensions => { - peadm::oid('pp_auth_role') => 'pe_compiler', + peadm::oid('pp_auth_role') => 'legacy_compiler', peadm::oid('peadm_availability_group') => 'A', - peadm::oid('peadm_legacy_compiler') => 'true', }, ) }, @@ -242,9 +239,8 @@ run_plan('peadm::modify_certificate', $legacy_compiler_b_targets, primary_host => $primary_target, add_extensions => { - peadm::oid('pp_auth_role') => 'pe_compiler', + peadm::oid('pp_auth_role') => 'legacy_compiler', peadm::oid('peadm_availability_group') => 'B', - peadm::oid('peadm_legacy_compiler') => 'true', }, ) }, @@ -283,6 +279,16 @@ include peadm::setup::convert_node_manager } + + # Unpin legacy compilers from PE Master group + if $legacy_compiler_targets { + $legacy_compiler_targets.each |$target| { + run_task('peadm::node_group_unpin', $primary_target, + node_certname => $target.peadm::certname(), + group_name => 'PE Master', + ) + } + } } else { # lint:ignore:strict_indent diff --git a/plans/convert_compiler_to_legacy.pp b/plans/convert_compiler_to_legacy.pp index c75924bd..55a86c3c 100644 --- a/plans/convert_compiler_to_legacy.pp +++ b/plans/convert_compiler_to_legacy.pp @@ -102,7 +102,7 @@ run_plan('peadm::modify_certificate', $compiler_targets, primary_host => $primary_target, add_extensions => { - peadm::oid('peadm_legacy_compiler') => 'false', + peadm::oid('pp_auth_role') => 'legacy_compiler', }, ) }, @@ -110,9 +110,8 @@ run_plan('peadm::modify_certificate', $legacy_compiler_a_targets, primary_host => $primary_target, add_extensions => { - peadm::oid('pp_auth_role') => 'pe_compiler', + peadm::oid('pp_auth_role') => 'legacy_compiler', peadm::oid('peadm_availability_group') => 'A', - peadm::oid('peadm_legacy_compiler') => 'true', }, ) }, @@ -120,9 +119,8 @@ run_plan('peadm::modify_certificate', $legacy_compiler_b_targets, primary_host => $primary_target, add_extensions => { - peadm::oid('pp_auth_role') => 'pe_compiler', + peadm::oid('pp_auth_role') => 'legacy_compiler', peadm::oid('peadm_availability_group') => 'B', - peadm::oid('peadm_legacy_compiler') => 'true', }, ) }, diff --git a/plans/subplans/component_install.pp b/plans/subplans/component_install.pp index c117ccb6..5929dd17 100644 --- a/plans/subplans/component_install.pp +++ b/plans/subplans/component_install.pp @@ -21,13 +21,11 @@ $certificate_extensions = { peadm::oid('pp_auth_role') => 'pe_compiler', peadm::oid('peadm_availability_group') => $avail_group_letter, - peadm::oid('peadm_legacy_compiler') => false, } } elsif $role == 'pe_compiler_legacy' { $certificate_extensions = { - peadm::oid('pp_auth_role') => 'pe_compiler', + peadm::oid('pp_auth_role') => 'legacy_compiler', peadm::oid('peadm_availability_group') => $avail_group_letter, - peadm::oid('peadm_legacy_compiler') => true, } } else { $certificate_extensions = { diff --git a/plans/subplans/install.pp b/plans/subplans/install.pp index 693c056c..ed57ba5b 100644 --- a/plans/subplans/install.pp +++ b/plans/subplans/install.pp @@ -287,7 +287,6 @@ extension_requests => { peadm::oid('pp_auth_role') => 'pe_compiler', peadm::oid('peadm_availability_group') => 'A', - peadm::oid('peadm_legacy_compiler') => 'false', } ) }, @@ -296,25 +295,22 @@ extension_requests => { peadm::oid('pp_auth_role') => 'pe_compiler', peadm::oid('peadm_availability_group') => 'B', - peadm::oid('peadm_legacy_compiler') => 'false', } ) }, background('compiler-a-csr.yaml') || { run_plan('peadm::util::insert_csr_extension_requests', $legacy_a_targets, extension_requests => { - peadm::oid('pp_auth_role') => 'pe_compiler', + peadm::oid('pp_auth_role') => 'legacy_compiler', peadm::oid('peadm_availability_group') => 'A', - peadm::oid('peadm_legacy_compiler') => 'true', } ) }, background('compiler-b-csr.yaml') || { run_plan('peadm::util::insert_csr_extension_requests', $legacy_b_targets, extension_requests => { - peadm::oid('pp_auth_role') => 'pe_compiler', + peadm::oid('pp_auth_role') => 'legacy_compiler', peadm::oid('peadm_availability_group') => 'B', - peadm::oid('peadm_legacy_compiler') => 'true', } ) }, diff --git a/plans/update_compiler_extensions.pp b/plans/update_compiler_extensions.pp index 784f919e..ff17e89f 100644 --- a/plans/update_compiler_extensions.pp +++ b/plans/update_compiler_extensions.pp @@ -7,11 +7,6 @@ $primary_target = peadm::get_targets($primary_host, 1) $host_targets = peadm::get_targets($compiler_hosts) - run_plan('peadm::modify_certificate', $host_targets, - primary_host => $primary_target, - add_extensions => { peadm::oid('peadm_legacy_compiler') => String($legacy) }, - ) - run_task('peadm::puppet_runonce', $primary_target) run_task('peadm::puppet_runonce', $host_targets) diff --git a/plans/upgrade.pp b/plans/upgrade.pp index 15b240fe..d009077a 100644 --- a/plans/upgrade.pp +++ b/plans/upgrade.pp @@ -172,8 +172,8 @@ $compiler_m1_nonlegacy_targets = $compiler_targets.filter |$target| { ($cert_extensions.dig($target.peadm::certname, peadm::oid('peadm_availability_group')) == $cert_extensions.dig($primary_target[0].peadm::certname, peadm::oid('peadm_availability_group'))) and - ($cert_extensions.dig($target.peadm::certname, peadm::oid('peadm_legacy_compiler')) - == 'false') + ($cert_extensions.dig($target.peadm::certname, peadm::oid('pp_auth_role')) + == 'pe_compiler') } $compiler_m2_targets = $compiler_targets.filter |$target| { @@ -184,8 +184,8 @@ $compiler_m2_nonlegacy_targets = $compiler_targets.filter |$target| { ($cert_extensions.dig($target.peadm::certname, peadm::oid('peadm_availability_group')) == $cert_extensions.dig($replica_target[0].peadm::certname, peadm::oid('peadm_availability_group'))) and - ($cert_extensions.dig($target.peadm::certname, peadm::oid('peadm_legacy_compiler')) - == 'false') + ($cert_extensions.dig($target.peadm::certname, peadm::oid('pp_auth_role')) + == 'pe_compiler') } peadm::plan_step('preparation') || { diff --git a/spec/plans/convert_spec.rb b/spec/plans/convert_spec.rb index 39ec7367..e3c702f0 100644 --- a/spec/plans/convert_spec.rb +++ b/spec/plans/convert_spec.rb @@ -9,7 +9,7 @@ end let(:params) do - { 'primary_host' => 'primary' } + { 'primary_host' => 'primary', 'legacy_compilers' => ['legacy_compiler'] } end it 'single primary no dr valid' do @@ -21,6 +21,8 @@ expect_task('peadm::cert_data').return_for_targets('primary' => trustedjson) expect_task('peadm::read_file').always_return({ 'content' => '2021.7.9' }) expect_task('peadm::get_group_rules').return_for_targets('primary' => { '_output' => '{"rules": []}' }) + expect_task('peadm::node_group_unpin').with_targets('primary').with_params({ 'node_certname' => 'legacy_compiler', 'group_name' => 'PE Master' }) + expect_task('peadm::check_legacy_compilers').with_targets('primary').with_params({ 'legacy_compilers' => 'legacy_compiler' }).return_for_targets('primary' => { '_output' => '' }) # For some reason, expect_plan() was not working?? allow_plan('peadm::modify_certificate').always_return({}) diff --git a/tasks/get_peadm_config.rb b/tasks/get_peadm_config.rb index 9eb3aa02..4bfffd35 100755 --- a/tasks/get_peadm_config.rb +++ b/tasks/get_peadm_config.rb @@ -22,7 +22,7 @@ def execute! def config # Compute values - primary = groups.pinned('PE Master') + primary = groups.pinned('PE Certificate Authority') replica = groups.pinned('PE HA Replica') server_a = server('puppet/server', 'A', [primary, replica].compact) server_b = server('puppet/server', 'B', [primary, replica].compact) @@ -94,8 +94,7 @@ def groups def compilers @compilers ||= pdb_query('inventory[certname,trusted.extensions] { - trusted.extensions.pp_auth_role = "pe_compiler" and - trusted.extensions."1.3.6.1.4.1.34380.1.1.9814" = "false" + trusted.extensions.pp_auth_role = "pe_compiler" }').map do |c| { 'certname' => c['certname'], @@ -108,8 +107,7 @@ def compilers def legacy_compilers @legacy_compilers ||= pdb_query('inventory[certname,trusted.extensions] { - trusted.extensions.pp_auth_role = "pe_compiler" and - trusted.extensions."1.3.6.1.4.1.34380.1.1.9814" = "true" + trusted.extensions.pp_auth_role = "legacy_compiler" }').map do |c| { 'certname' => c['certname'], diff --git a/tasks/node_group_unpin.json b/tasks/node_group_unpin.json new file mode 100644 index 00000000..806a1abf --- /dev/null +++ b/tasks/node_group_unpin.json @@ -0,0 +1,17 @@ +{ + "description": "Unpins a node from a specified PE node group", + "parameters": { + "node_certname": { + "type": "String", + "description": "The certname of the node to unpin" + }, + "group_name": { + "type": "String", + "description": "The name of the node group to unpin the node from" + } + }, + "input_method": "stdin", + "implementations": [ + {"name": "node_group_unpin.rb"} + ] +} \ No newline at end of file diff --git a/tasks/node_group_unpin.rb b/tasks/node_group_unpin.rb new file mode 100755 index 00000000..c1594f62 --- /dev/null +++ b/tasks/node_group_unpin.rb @@ -0,0 +1,117 @@ +#!/opt/puppetlabs/puppet/bin/ruby +# frozen_string_literal: true + +require 'json' +require 'yaml' +require 'net/https' +require 'puppet' + +# NodeGroupUnpin task class +class NodeGroupUnpin + def initialize(params) + @params = params + raise "Missing required parameter 'node_certname'" unless @params['node_certname'] + raise "Missing required parameter 'group_name'" unless @params['group_name'] + @auth = YAML.load_file('/etc/puppetlabs/puppet/classifier.yaml') + rescue Errno::ENOENT + raise 'Could not find classifier.yaml at /etc/puppetlabs/puppet/classifier.yaml' + end + + def https_client + client = Net::HTTP.new(Puppet.settings[:certname], 4433) + client.use_ssl = true + client.cert = @cert ||= OpenSSL::X509::Certificate.new(File.read(Puppet.settings[:hostcert])) + client.key = @key ||= OpenSSL::PKey::RSA.new(File.read(Puppet.settings[:hostprivkey])) + client.verify_mode = OpenSSL::SSL::VERIFY_PEER + client.ca_file = Puppet.settings[:localcacert] + client + end + + def groups + @groups ||= begin + net = https_client + res = net.get('/classifier-api/v1/groups') + + unless res.code == '200' + raise "Failed to fetch groups: HTTP #{res.code} - #{res.body}" + end + + NodeGroup.new(JSON.parse(res.body)) + rescue JSON::ParserError => e + raise "Invalid JSON response from server: #{e.message}" + rescue StandardError => e + raise "Error fetching groups: #{e.message}" + end + end + + def unpin_node(group, node) + raise 'Invalid group object' unless group.is_a?(Hash) && group['id'] && group['name'] + + net = https_client + begin + data = { "nodes": [node] }.to_json + url = "/classifier-api/v1/groups/#{group['id']}/unpin" + + req = Net::HTTP::Post.new(url) + req['Content-Type'] = 'application/json' + req.body = data + + res = net.request(req) + + case res.code + when '204' + puts "Successfully unpinned node '#{node}' from group '#{group['name']}'" + else + begin + error_body = JSON.parse(res.body.to_s) + raise "Failed to unpin node: #{error_body['kind'] || error_body}" + rescue JSON::ParserError + raise "Invalid response from server (status #{res.code}): #{res.body}" + end + end + rescue StandardError => e + raise "Error during unpin request: #{e.message}" + end + end + + # Utility class to aid in retrieving useful information from the node group + # data + class NodeGroup + attr_reader :data + + def initialize(data) + @data = data + end + + # Aids in digging into node groups by name, rather than UUID + def dig(name, *args) + group = @data.find { |obj| obj['name'] == name } + if group.nil? + nil + elsif args.empty? + group + else + group.dig(*args) + end + end + end + + def execute! + group_name = @params['group_name'] + node_certname = @params['node_certname'] + group = groups.dig(group_name) + if group + unpin_node(group, node_certname) + puts "Unpinned #{node_certname} from #{group_name}" + else + puts "Group #{group_name} not found" + end + end +end + +# Run the task unless an environment flag has been set +unless ENV['RSPEC_UNIT_TEST_MODE'] + Puppet.initialize_settings + task = NodeGroupUnpin.new(JSON.parse(STDIN.read)) + task.execute! +end From a0db439216081963df2c7cdabdd7379646b5e90e Mon Sep 17 00:00:00 2001 From: petergmurphy Date: Thu, 30 Jan 2025 17:06:54 +0000 Subject: [PATCH 2/4] Make node_group_unpin accept an array of certnames --- REFERENCE.md | 12 ++++++------ plans/convert.pp | 10 ++++------ spec/plans/convert_spec.rb | 2 +- tasks/node_group_unpin.json | 10 +++++----- tasks/node_group_unpin.rb | 17 +++++++++-------- 5 files changed, 25 insertions(+), 26 deletions(-) diff --git a/REFERENCE.md b/REFERENCE.md index 3fc16531..c4454500 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -74,7 +74,7 @@ * [`infrastatus`](#infrastatus): Runs puppet infra status and returns the output * [`mkdir_p_file`](#mkdir_p_file): Create a file with the specified content at the specified location * [`mv`](#mv): Wrapper task for mv command -* [`node_group_unpin`](#node_group_unpin): Unpins a node from a specified PE node group +* [`node_group_unpin`](#node_group_unpin): Unpins nodes from a specified PE node group * [`os_identification`](#os_identification): Return the operating system runnin gon the target as a string * [`pe_install`](#pe_install): Install Puppet Enterprise from a tarball * [`pe_ldap_config`](#pe_ldap_config): Set the ldap config in the PE console @@ -1327,23 +1327,23 @@ New path of file ### `node_group_unpin` -Unpins a node from a specified PE node group +Unpins nodes from a specified PE node group **Supports noop?** false #### Parameters -##### `node_certname` +##### `node_certnames` -Data type: `String` +Data type: `Array[String]` -The certname of the node to unpin +The certnames of the nodes to unpin ##### `group_name` Data type: `String` -The name of the node group to unpin the node from +The name of the node group to unpin the nodes from ### `os_identification` diff --git a/plans/convert.pp b/plans/convert.pp index 11ee1209..5277abe6 100644 --- a/plans/convert.pp +++ b/plans/convert.pp @@ -282,12 +282,10 @@ # Unpin legacy compilers from PE Master group if $legacy_compiler_targets { - $legacy_compiler_targets.each |$target| { - run_task('peadm::node_group_unpin', $primary_target, - node_certname => $target.peadm::certname(), - group_name => 'PE Master', - ) - } + run_task('peadm::node_group_unpin', $primary_target, + node_certnames => $legacy_compiler_targets.map |$target| { $target.peadm::certname() }, + group_name => 'PE Master', + ) } } else { diff --git a/spec/plans/convert_spec.rb b/spec/plans/convert_spec.rb index e3c702f0..96bacfe1 100644 --- a/spec/plans/convert_spec.rb +++ b/spec/plans/convert_spec.rb @@ -21,7 +21,7 @@ expect_task('peadm::cert_data').return_for_targets('primary' => trustedjson) expect_task('peadm::read_file').always_return({ 'content' => '2021.7.9' }) expect_task('peadm::get_group_rules').return_for_targets('primary' => { '_output' => '{"rules": []}' }) - expect_task('peadm::node_group_unpin').with_targets('primary').with_params({ 'node_certname' => 'legacy_compiler', 'group_name' => 'PE Master' }) + expect_task('peadm::node_group_unpin').with_targets('primary').with_params({ 'node_certnames' => ['legacy_compiler'], 'group_name' => 'PE Master' }) expect_task('peadm::check_legacy_compilers').with_targets('primary').with_params({ 'legacy_compilers' => 'legacy_compiler' }).return_for_targets('primary' => { '_output' => '' }) # For some reason, expect_plan() was not working?? diff --git a/tasks/node_group_unpin.json b/tasks/node_group_unpin.json index 806a1abf..c94f3654 100644 --- a/tasks/node_group_unpin.json +++ b/tasks/node_group_unpin.json @@ -1,13 +1,13 @@ { - "description": "Unpins a node from a specified PE node group", + "description": "Unpins nodes from a specified PE node group", "parameters": { - "node_certname": { - "type": "String", - "description": "The certname of the node to unpin" + "node_certnames": { + "type": "Array[String]", + "description": "The certnames of the nodes to unpin" }, "group_name": { "type": "String", - "description": "The name of the node group to unpin the node from" + "description": "The name of the node group to unpin the nodes from" } }, "input_method": "stdin", diff --git a/tasks/node_group_unpin.rb b/tasks/node_group_unpin.rb index c1594f62..2a2e5637 100755 --- a/tasks/node_group_unpin.rb +++ b/tasks/node_group_unpin.rb @@ -10,7 +10,8 @@ class NodeGroupUnpin def initialize(params) @params = params - raise "Missing required parameter 'node_certname'" unless @params['node_certname'] + raise "Missing required parameter 'node_certnames'" unless @params['node_certnames'] + raise "'node_certnames' must be an array" unless @params['node_certnames'].is_a?(Array) raise "Missing required parameter 'group_name'" unless @params['group_name'] @auth = YAML.load_file('/etc/puppetlabs/puppet/classifier.yaml') rescue Errno::ENOENT @@ -44,12 +45,12 @@ def groups end end - def unpin_node(group, node) + def unpin_node(group, nodes) raise 'Invalid group object' unless group.is_a?(Hash) && group['id'] && group['name'] net = https_client begin - data = { "nodes": [node] }.to_json + data = { "nodes": nodes }.to_json url = "/classifier-api/v1/groups/#{group['id']}/unpin" req = Net::HTTP::Post.new(url) @@ -60,11 +61,11 @@ def unpin_node(group, node) case res.code when '204' - puts "Successfully unpinned node '#{node}' from group '#{group['name']}'" + puts "Successfully unpinned nodes '#{nodes.join(', ')}' from group '#{group['name']}'" else begin error_body = JSON.parse(res.body.to_s) - raise "Failed to unpin node: #{error_body['kind'] || error_body}" + raise "Failed to unpin nodes: #{error_body['kind'] || error_body}" rescue JSON::ParserError raise "Invalid response from server (status #{res.code}): #{res.body}" end @@ -98,11 +99,11 @@ def dig(name, *args) def execute! group_name = @params['group_name'] - node_certname = @params['node_certname'] + node_certnames = @params['node_certnames'] group = groups.dig(group_name) if group - unpin_node(group, node_certname) - puts "Unpinned #{node_certname} from #{group_name}" + unpin_node(group, node_certnames) + puts "Unpinned #{node_certnames.join(', ')} from #{group_name}" else puts "Group #{group_name} not found" end From 81e986a8e9cbf7e6bb19f42ce3939b2660725154 Mon Sep 17 00:00:00 2001 From: petergmurphy Date: Wed, 5 Feb 2025 09:44:30 +0000 Subject: [PATCH 3/4] Add task to update PE Master group rules This commit introduces a new private task to update the AND conditional for the pe_compiler auth role in the PE Master node group, changing it to regex match for any *_compiler role. The task ensures that the group rules are simplified and display more correctly on the PE console. --- manifests/setup/legacy_compiler_group.pp | 24 +++-- manifests/setup/node_manager.pp | 10 +- plans/convert.pp | 6 +- plans/convert_compiler_to_legacy.pp | 6 +- plans/install.pp | 2 + plans/subplans/component_install.pp | 2 +- plans/subplans/install.pp | 4 +- plans/upgrade.pp | 2 + spec/plans/convert_spec.rb | 6 +- tasks/update_pe_master_rules.json | 8 ++ tasks/update_pe_master_rules.rb | 117 +++++++++++++++++++++++ 11 files changed, 156 insertions(+), 31 deletions(-) create mode 100644 tasks/update_pe_master_rules.json create mode 100755 tasks/update_pe_master_rules.rb diff --git a/manifests/setup/legacy_compiler_group.pp b/manifests/setup/legacy_compiler_group.pp index 8c31a9c0..5534ae49 100644 --- a/manifests/setup/legacy_compiler_group.pp +++ b/manifests/setup/legacy_compiler_group.pp @@ -9,11 +9,10 @@ } node_group { 'PE Legacy Compiler': - ensure => 'present', - parent => 'PE Master', - purge_behavior => 'classes', - rule => ['=', ['trusted', 'extensions', 'pp_auth_role'], 'legacy_compiler'], - classes => { + ensure => 'present', + parent => 'PE Infrastructure', + rule => ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler_legacy'], + classes => { 'puppet_enterprise::profile::master' => { 'puppetdb_host' => [$internal_compiler_a_pool_address, $internal_compiler_b_pool_address].filter |$_| { $_ }, 'puppetdb_port' => [8081], @@ -24,20 +23,19 @@ } node_group { 'PE Legacy Compiler Group A': - ensure => 'present', - parent => 'PE Legacy Compiler', - purge_behavior => 'classes', - rule => ['and', - ['=', ['trusted', 'extensions', 'pp_auth_role'], 'legacy_compiler'], + ensure => 'present', + parent => 'PE Legacy Compiler', + rule => ['and', + ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler_legacy'], ['=', ['trusted', 'extensions', peadm::oid('peadm_availability_group')], 'A'], ], - classes => { + classes => { 'puppet_enterprise::profile::master' => { 'puppetdb_host' => [$internal_compiler_b_pool_address, $internal_compiler_a_pool_address].filter |$_| { $_ }, 'puppetdb_port' => [8081], }, }, - data => { + data => { 'puppet_enterprise::profile::master::puppetdb' => { 'ha_enabled_replicas' => [], }, @@ -49,7 +47,7 @@ parent => 'PE Legacy Compiler', purge_behavior => 'classes', rule => ['and', - ['=', ['trusted', 'extensions', 'pp_auth_role'], 'legacy_compiler'], + ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler_legacy'], ['=', ['trusted', 'extensions', peadm::oid('peadm_availability_group')], 'B'], ], classes => { diff --git a/manifests/setup/node_manager.pp b/manifests/setup/node_manager.pp index 5334a3f1..fad61950 100644 --- a/manifests/setup/node_manager.pp +++ b/manifests/setup/node_manager.pp @@ -77,10 +77,6 @@ parent => 'PE Infrastructure', data => $compiler_pool_address_data, variables => { 'pe_master' => true }, - rule => ['or', - ['=', ['trusted', 'extensions', 'pp_auth_role'], 'legacy_compiler'], - ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], - ], } # PE Compiler group comes from default PE and already has the pe compiler role @@ -205,7 +201,7 @@ node_group { 'PE Legacy Compiler': parent => 'PE Master', - rule => ['=', ['trusted', 'extensions', 'pp_auth_role'], 'legacy_compiler'], + rule => ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler_legacy'], classes => { 'puppet_enterprise::profile::master' => { 'puppetdb_host' => [$internal_compiler_a_pool_address, $internal_compiler_b_pool_address].filter |$_| { $_ }, @@ -220,7 +216,7 @@ ensure => 'present', parent => 'PE Legacy Compiler', rule => ['and', - ['=', ['trusted', 'extensions', 'pp_auth_role'], 'legacy_compiler'], + ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler_legacy'], ['=', ['trusted', 'extensions', peadm::oid('peadm_availability_group')], 'A'], ], classes => { @@ -243,7 +239,7 @@ ensure => 'present', parent => 'PE Legacy Compiler', rule => ['and', - ['=', ['trusted', 'extensions', 'pp_auth_role'], 'legacy_compiler'], + ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler_legacy'], ['=', ['trusted', 'extensions', peadm::oid('peadm_availability_group')], 'B'], ], classes => { diff --git a/plans/convert.pp b/plans/convert.pp index 5277abe6..ef431dc3 100644 --- a/plans/convert.pp +++ b/plans/convert.pp @@ -230,7 +230,7 @@ run_plan('peadm::modify_certificate', $legacy_compiler_a_targets, primary_host => $primary_target, add_extensions => { - peadm::oid('pp_auth_role') => 'legacy_compiler', + peadm::oid('pp_auth_role') => 'pe_compiler_legacy', peadm::oid('peadm_availability_group') => 'A', }, ) @@ -239,7 +239,7 @@ run_plan('peadm::modify_certificate', $legacy_compiler_b_targets, primary_host => $primary_target, add_extensions => { - peadm::oid('pp_auth_role') => 'legacy_compiler', + peadm::oid('pp_auth_role') => 'pe_compiler_legacy', peadm::oid('peadm_availability_group') => 'B', }, ) @@ -333,5 +333,7 @@ # lint:endignore } + run_task('peadm::update_pe_master_rules', $primary_target) + return("Conversion to peadm Puppet Enterprise ${arch['architecture']} completed.") } diff --git a/plans/convert_compiler_to_legacy.pp b/plans/convert_compiler_to_legacy.pp index 55a86c3c..2f662013 100644 --- a/plans/convert_compiler_to_legacy.pp +++ b/plans/convert_compiler_to_legacy.pp @@ -102,7 +102,7 @@ run_plan('peadm::modify_certificate', $compiler_targets, primary_host => $primary_target, add_extensions => { - peadm::oid('pp_auth_role') => 'legacy_compiler', + peadm::oid('pp_auth_role') => 'pe_compiler_legacy', }, ) }, @@ -110,7 +110,7 @@ run_plan('peadm::modify_certificate', $legacy_compiler_a_targets, primary_host => $primary_target, add_extensions => { - peadm::oid('pp_auth_role') => 'legacy_compiler', + peadm::oid('pp_auth_role') => 'pe_compiler_legacy', peadm::oid('peadm_availability_group') => 'A', }, ) @@ -119,7 +119,7 @@ run_plan('peadm::modify_certificate', $legacy_compiler_b_targets, primary_host => $primary_target, add_extensions => { - peadm::oid('pp_auth_role') => 'legacy_compiler', + peadm::oid('pp_auth_role') => 'pe_compiler_legacy', peadm::oid('peadm_availability_group') => 'B', }, ) diff --git a/plans/install.pp b/plans/install.pp index 4e55e0be..7ce03cf0 100644 --- a/plans/install.pp +++ b/plans/install.pp @@ -143,6 +143,8 @@ final_agent_state => $final_agent_state, ) + run_task('peadm::update_pe_master_rules', $primary_host) + # Return a string banner reporting on what was done return([$install_result, $configure_result]) } diff --git a/plans/subplans/component_install.pp b/plans/subplans/component_install.pp index 5929dd17..2fad74e8 100644 --- a/plans/subplans/component_install.pp +++ b/plans/subplans/component_install.pp @@ -24,7 +24,7 @@ } } elsif $role == 'pe_compiler_legacy' { $certificate_extensions = { - peadm::oid('pp_auth_role') => 'legacy_compiler', + peadm::oid('pp_auth_role') => 'pe_compiler_legacy', peadm::oid('peadm_availability_group') => $avail_group_letter, } } else { diff --git a/plans/subplans/install.pp b/plans/subplans/install.pp index ed57ba5b..96af47f0 100644 --- a/plans/subplans/install.pp +++ b/plans/subplans/install.pp @@ -301,7 +301,7 @@ background('compiler-a-csr.yaml') || { run_plan('peadm::util::insert_csr_extension_requests', $legacy_a_targets, extension_requests => { - peadm::oid('pp_auth_role') => 'legacy_compiler', + peadm::oid('pp_auth_role') => 'pe_compiler_legacy', peadm::oid('peadm_availability_group') => 'A', } ) @@ -309,7 +309,7 @@ background('compiler-b-csr.yaml') || { run_plan('peadm::util::insert_csr_extension_requests', $legacy_b_targets, extension_requests => { - peadm::oid('pp_auth_role') => 'legacy_compiler', + peadm::oid('pp_auth_role') => 'pe_compiler_legacy', peadm::oid('peadm_availability_group') => 'B', } ) diff --git a/plans/upgrade.pp b/plans/upgrade.pp index d009077a..43f565b7 100644 --- a/plans/upgrade.pp +++ b/plans/upgrade.pp @@ -440,5 +440,7 @@ peadm::check_version_and_known_hosts($current_pe_version, $_version, $r10k_known_hosts) + run_task('peadm::update_pe_master_rules', $primary_target) + return("Upgrade of Puppet Enterprise ${arch['architecture']} completed.") } diff --git a/spec/plans/convert_spec.rb b/spec/plans/convert_spec.rb index 96bacfe1..161123cc 100644 --- a/spec/plans/convert_spec.rb +++ b/spec/plans/convert_spec.rb @@ -9,7 +9,7 @@ end let(:params) do - { 'primary_host' => 'primary', 'legacy_compilers' => ['legacy_compiler'] } + { 'primary_host' => 'primary', 'legacy_compilers' => ['pe_compiler_legacy'] } end it 'single primary no dr valid' do @@ -21,8 +21,8 @@ expect_task('peadm::cert_data').return_for_targets('primary' => trustedjson) expect_task('peadm::read_file').always_return({ 'content' => '2021.7.9' }) expect_task('peadm::get_group_rules').return_for_targets('primary' => { '_output' => '{"rules": []}' }) - expect_task('peadm::node_group_unpin').with_targets('primary').with_params({ 'node_certnames' => ['legacy_compiler'], 'group_name' => 'PE Master' }) - expect_task('peadm::check_legacy_compilers').with_targets('primary').with_params({ 'legacy_compilers' => 'legacy_compiler' }).return_for_targets('primary' => { '_output' => '' }) + expect_task('peadm::node_group_unpin').with_targets('primary').with_params({ 'node_certnames' => ['pe_compiler_legacy'], 'group_name' => 'PE Master' }) + expect_task('peadm::check_legacy_compilers').with_targets('primary').with_params({ 'legacy_compilers' => 'pe_compiler_legacy' }).return_for_targets('primary' => { '_output' => '' }) # For some reason, expect_plan() was not working?? allow_plan('peadm::modify_certificate').always_return({}) diff --git a/tasks/update_pe_master_rules.json b/tasks/update_pe_master_rules.json new file mode 100644 index 00000000..664e89c3 --- /dev/null +++ b/tasks/update_pe_master_rules.json @@ -0,0 +1,8 @@ +{ + "description": "Updates the PE Master group rules to replace pe_compiler with a regex match for any pe_compiler role", + "input_method": "stdin", + "private": true, + "implementations": [ + {"name": "update_pe_master_rules.rb"} + ] +} \ No newline at end of file diff --git a/tasks/update_pe_master_rules.rb b/tasks/update_pe_master_rules.rb new file mode 100755 index 00000000..85219daf --- /dev/null +++ b/tasks/update_pe_master_rules.rb @@ -0,0 +1,117 @@ +#!/opt/puppetlabs/puppet/bin/ruby +# frozen_string_literal: true + +require 'json' +require 'net/https' +require 'puppet' + +# UpdatePeMasterRules task class +class UpdatePeMasterRules + def initialize(params) + @params = params + end + + def https_client + client = Net::HTTP.new(Puppet.settings[:certname], 4433) + client.use_ssl = true + client.cert = @cert ||= OpenSSL::X509::Certificate.new(File.read(Puppet.settings[:hostcert])) + client.key = @key ||= OpenSSL::PKey::RSA.new(File.read(Puppet.settings[:hostprivkey])) + client.verify_mode = OpenSSL::SSL::VERIFY_PEER + client.ca_file = Puppet.settings[:localcacert] + client + end + + def get_pe_master_group_id + net = https_client + res = net.get('/classifier-api/v1/groups') + + unless res.code == '200' + raise "Failed to fetch groups: HTTP #{res.code} - #{res.body}" + end + + groups = JSON.parse(res.body) + pe_master_group = groups.find { |group| group['name'] == 'PE Master' } + + raise "Could not find PE Master group" unless pe_master_group + pe_master_group['id'] + rescue JSON::ParserError => e + raise "Invalid JSON response from server: #{e.message}" + rescue StandardError => e + raise "Error fetching PE Master group ID: #{e.message}" + end + + def get_current_rules(group_id) + net = https_client + url = "/classifier-api/v1/groups/#{group_id}/rules" + req = Net::HTTP::Get.new(url) + res = net.request(req) + + unless res.code == '200' + raise "Failed to fetch rules: HTTP #{res.code} - #{res.body}" + end + + JSON.parse(res.body)['rule'] + rescue JSON::ParserError => e + raise "Invalid JSON response from server: #{e.message}" + rescue StandardError => e + raise "Error fetching rules: #{e.message}" + end + + def transform_rule(rule) + return rule unless rule.is_a?(Array) + + if rule[0] == '=' && + rule[1].is_a?(Array) && + rule[1] == ['trusted', 'extensions', 'pp_auth_role'] && + rule[2] == 'pe_compiler' + return ['~', ['trusted', 'extensions', 'pp_auth_role'], '^pe_compiler(?:_legacy)?$'] + end + + # Recursively transform nested rules + rule.map { |element| transform_rule(element) } + end + + def update_rules(group_id) + net = https_client + begin + current_rules = get_current_rules(group_id) + + # Transform rules recursively to handle nested structures + new_rules = transform_rule(current_rules) + + # Update the group with the modified rules + url = "/classifier-api/v1/groups/#{group_id}" + req = Net::HTTP::Post.new(url) + req['Content-Type'] = 'application/json' + req.body = { rule: new_rules }.to_json + + res = net.request(req) + + case res.code + when '200', '201', '204' + puts "Successfully transformed pe_compiler rule to use regex match for *_compiler roles in group #{group_id}" + else + begin + error_body = JSON.parse(res.body.to_s) + raise "Failed to update rules: #{error_body['kind'] || error_body}" + rescue JSON::ParserError + raise "Invalid response from server (status #{res.code}): #{res.body}" + end + end + rescue StandardError => e + raise "Error during rules update: #{e.message}" + end + end + + def execute! + group_id = get_pe_master_group_id + update_rules(group_id) + end +end + +# Run the task unless an environment flag has been set +unless ENV['RSPEC_UNIT_TEST_MODE'] + Puppet.initialize_settings + task = UpdatePeMasterRules.new(JSON.parse(STDIN.read)) + task.execute! +end \ No newline at end of file From 55d852fb1509857a75273bcc1d9c2efad28ff9d5 Mon Sep 17 00:00:00 2001 From: petergmurphy Date: Tue, 11 Feb 2025 15:45:30 +0000 Subject: [PATCH 4/4] Migrate to new method for legacy compiler checks --- plans/upgrade.pp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/plans/upgrade.pp b/plans/upgrade.pp index 43f565b7..a591bb5f 100644 --- a/plans/upgrade.pp +++ b/plans/upgrade.pp @@ -135,6 +135,26 @@ peadm::assert_supported_pe_version($_version, $permit_unsafe_versions) + # Gather certificate extension information from all systems + $cert_extensions_temp = run_task('peadm::cert_data', $all_targets).reduce({}) |$memo,$result| { + $memo + { $result.target.peadm::certname => $result['extensions'] } + } + + # Add legacy compiler role to compilers that are missing it + $legacy_compiler_targets = $cert_extensions_temp.filter |$name,$exts| { + ($name in $compiler_targets.map |$t| { $t.name }) and + ($exts[peadm::oid('peadm_legacy_compiler')] != undef) and + ($exts[peadm::oid('peadm_legacy_compiler')] == 'true') and + ($exts['pp_auth_role'] != 'pe_compiler_legacy') + }.keys + + run_plan('peadm::modify_certificate', $legacy_compiler_targets, + primary_host => $primary_target, + add_extensions => { + 'pp_auth_role' => 'pe_compiler_legacy', + }, + ) + # Gather certificate extension information from all systems $cert_extensions = run_task('peadm::cert_data', $all_targets).reduce({}) |$memo,$result| { $memo + { $result.target.peadm::certname => $result['extensions'] }