Puppet Plan: peadm::convert

Defined in:
plans/convert.pp

Summary

Convert an existing PE cluster to a PEAdm-managed cluster

Overview

This plan sets required certificate extensions on PE nodes, and configures the required PE node groups to make an existing cluster compatible with management using PEAdm.

Parameters:

  • begin_at_step (Optional[Peadm::ConvertSteps]) (defaults to: undef) —

    The step where the plan should start. If not set, it will start at the beginning

  • primary_host (Peadm::SingleTargetSpec)
  • replica_host (Optional[Peadm::SingleTargetSpec]) (defaults to: undef)
  • compiler_hosts (Optional[TargetSpec]) (defaults to: undef)
  • legacy_compilers (Optional[TargetSpec]) (defaults to: undef)
  • primary_postgresql_host (Optional[Peadm::SingleTargetSpec]) (defaults to: undef)
  • replica_postgresql_host (Optional[Peadm::SingleTargetSpec]) (defaults to: undef)
  • compiler_pool_address (String) (defaults to: $primary_host)
  • internal_compiler_a_pool_address (Optional[String]) (defaults to: undef)
  • internal_compiler_b_pool_address (Optional[String]) (defaults to: undef)
  • dns_alt_names (Array[String]) (defaults to: []) 


9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
 
# File 'plans/convert.pp', line 9

plan peadm::convert (
  # Standard
  Peadm::SingleTargetSpec           $primary_host,
  Optional[Peadm::SingleTargetSpec] $replica_host = undef,

  # Large
  Optional[TargetSpec]              $compiler_hosts = undef,
  Optional[TargetSpec]              $legacy_compilers = undef,

  # Extra Large
  Optional[Peadm::SingleTargetSpec] $primary_postgresql_host         = undef,
  Optional[Peadm::SingleTargetSpec] $replica_postgresql_host = undef,

  # Common Configuration
  String                            $compiler_pool_address            = $primary_host,
  Optional[String]                  $internal_compiler_a_pool_address = undef,
  Optional[String]                  $internal_compiler_b_pool_address = undef,
  Array[String]                     $dns_alt_names                    = [],

  Optional[Peadm::ConvertSteps] $begin_at_step = undef,
) {
  peadm::assert_supported_bolt_version()

  # TODO: read and validate convertable PE version

  # Convert inputs into targets.
  $primary_target                   = peadm::get_targets($primary_host, 1)
  $replica_target                   = peadm::get_targets($replica_host, 1)
  $replica_postgresql_target        = peadm::get_targets($replica_postgresql_host, 1)
  $compiler_targets                 = peadm::get_targets($compiler_hosts)
  $legacy_compiler_targets          = peadm::get_targets($legacy_compilers)
  $primary_postgresql_target        = peadm::get_targets($primary_postgresql_host, 1)

  $all_targets = peadm::flatten_compact([
      $primary_target,
      $replica_target,
      $replica_postgresql_target,
      $compiler_targets,
      $legacy_compiler_targets,
      $primary_postgresql_target,
  ])

  # Ensure input valid for a supported architecture
  $arch = peadm::assert_supported_architecture(
    $primary_host,
    $replica_host,
    $primary_postgresql_host,
    $replica_postgresql_host,
    $compiler_hosts,
    $legacy_compilers,
  )

  out::message('# Gathering information')

  # Get trusted fact information for all compilers. Use peadm::certname() as
  # the hash key because the apply block below will break trying to parse the
  # $compiler_extensions variable if it has Target-type hash keys.
  $cert_extensions = run_task('peadm::cert_data', $all_targets).reduce({}) |$memo,$result| {
    $memo + { $result.target.peadm::certname() => $result['extensions'] }
  }

  # Know what version of PE the current targets are
  $pe_version = run_task('peadm::read_file', $primary_target,
    path => '/opt/puppetlabs/server/pe_version',
  )[0][content].chomp

  # Figure out if this PE cluster has been configured with peadm or pe_xl
  # before
  $previously_configured_by_peadm = $all_targets.any |$target| {
    $exts = $cert_extensions[$target.peadm::certname()]
    $exts[peadm::oid('peadm_role')] or String($exts[peadm::oid('pp_role')]) =~ /pe_xl|peadm/
  }

  if (!$previously_configured_by_peadm and ($pe_version =~ SemVerRange('< 2019.7.0'))) {
# lint:ignore:strict_indent
    fail_plan(@("EOL"/L))
      PE cluster cannot be converted! PE cluster must be a deployment \
      created by pe_xl, by an older version of peadm, or be PE version \
      2019.7.0 or newer. Detected PE version ${pe_version}, and did not detect \
      signs that the deployment was previously created by peadm/pe_xl.
      | EOL
# lint:endignore
  }

  # Clusters A and B are used to divide PuppetDB availability for compilers. If
  # the compilers given already have peadm_availability_group facts designating
  # them A or B, use that. Otherwise, divide them by modulus of 2.
  if $arch['disaster-recovery'] {
    $compiler_a_targets = $compiler_targets.filter |$index,$target| {
      $exts = $cert_extensions[$target.peadm::certname()]
      if ($exts[peadm::oid('peadm_availability_group')] in ['A', 'B']) {
        $exts[peadm::oid('peadm_availability_group')] == 'A'
      }
      elsif ($exts[peadm::oid('pp_cluster')] in ['A', 'B']) {
        $exts[peadm::oid('pp_cluster')] == 'A'
      }
      else {
        $index % 2 == 0
      }
    }
    $compiler_b_targets = $compiler_targets.filter |$index,$target| {
      $exts = $cert_extensions[$target.peadm::certname()]
      if ($exts[peadm::oid('peadm_availability_group')] in ['A', 'B']) {
        $exts[peadm::oid('peadm_availability_group')] == 'B'
      }
      elsif ($exts[peadm::oid('pp_cluster')] in ['A', 'B']) {
        $exts[peadm::oid('pp_cluster')] == 'B'
      }
      else {
        $index % 2 != 0
      }
    }
    $legacy_compiler_a_targets = $legacy_compiler_targets.filter |$index,$target| {
      $exts = $cert_extensions[$target.peadm::certname()]
      if ($exts[peadm::oid('peadm_availability_group')] in ['A', 'B']) {
        $exts[peadm::oid('peadm_availability_group')] == 'A'
      }
      elsif ($exts[peadm::oid('pp_cluster')] in ['A', 'B']) {
        $exts[peadm::oid('pp_cluster')] == 'A'
      }
      else {
        $index % 2 == 0
      }
    }
    $legacy_compiler_b_targets = $legacy_compiler_targets.filter |$index,$target| {
      $exts = $cert_extensions[$target.peadm::certname()]
      if ($exts[peadm::oid('peadm_availability_group')] in ['A', 'B']) {
        $exts[peadm::oid('peadm_availability_group')] == 'B'
      }
      elsif ($exts[peadm::oid('pp_cluster')] in ['A', 'B']) {
        $exts[peadm::oid('pp_cluster')] == 'B'
      }
      else {
        $index % 2 != 0
      }
    }
  }
  else {
    $compiler_a_targets = $compiler_targets
    $compiler_b_targets = []
    $legacy_compiler_a_targets = $legacy_compiler_targets
    $legacy_compiler_b_targets = []
  }

  # Modify csr_attributes.yaml and insert the peadm-specific OIDs to identify
  # each server's role and availability group

  peadm::plan_step('modify-primary-cert') || {
    # If PE version is older than 2019.7
    if (versioncmp($pe_version, '2019.7.0') < 0) {
      apply($primary_target) {
        include peadm::setup::convert_pre20197
      }
    }

    run_plan('peadm::modify_certificate', $primary_target,
      primary_host   => $primary_target,
      add_extensions => {
        peadm::oid('peadm_role')               => 'puppet/server',
        peadm::oid('peadm_availability_group') => 'A',
      },
    )
  }

  peadm::plan_step('modify-infra-certs') || {
    # If the orchestrator is in use, get certs fully straightened up before
    # proceeding
    if $all_targets.any |$target| { $target.protocol == 'pcp' } {
      run_task('peadm::puppet_runonce', $primary_target)
      peadm::wait_until_service_ready('orchestrator-service', $primary_target)
      wait_until_available($all_targets, wait_time => 120)
    }

    # Kick off all the cert modification jobs in parallel
    $background_cert_jobs = [
      background('modify-replica-cert') || {
        run_plan('peadm::modify_certificate', $replica_target,
          primary_host   => $primary_target,
          add_extensions => {
            peadm::oid('peadm_role')               => 'puppet/server',
            peadm::oid('peadm_availability_group') => 'B',
          },
        )
      },
      background('modify-primary-postgresql-cert') || {
        run_plan('peadm::modify_certificate', $primary_postgresql_target,
          primary_host   => $primary_target,
          add_extensions => {
            peadm::oid('peadm_role')               => 'puppet/puppetdb-database',
            peadm::oid('peadm_availability_group') => 'A',
          },
        )
      },
      background('modify-replica-postgresql-cert') || {
        run_plan('peadm::modify_certificate', $replica_postgresql_target,
          primary_host   => $primary_target,
          add_extensions => {
            peadm::oid('peadm_role')               => 'puppet/puppetdb-database',
            peadm::oid('peadm_availability_group') => 'B',
          },
        )
      },
      background('modify-compilers-a-certs') || {
        run_plan('peadm::modify_certificate', $compiler_a_targets,
          primary_host   => $primary_target,
          add_extensions => {
            peadm::oid('pp_auth_role')             => 'pe_compiler',
            peadm::oid('peadm_availability_group') => 'A',
            peadm::oid('peadm_legacy_compiler')    => 'false',
          },
        )
      },
      background('modify-compilers-b-certs') || {
        run_plan('peadm::modify_certificate', $compiler_b_targets,
          primary_host   => $primary_target,
          add_extensions => {
            peadm::oid('pp_auth_role')             => 'pe_compiler',
            peadm::oid('peadm_availability_group') => 'B',
            peadm::oid('peadm_legacy_compiler')    => 'false',
          },
        )
      },
      background('modify-compilers-a-certs') || {
        run_plan('peadm::modify_certificate', $legacy_compiler_a_targets,
          primary_host   => $primary_target,
          add_extensions => {
            peadm::oid('pp_auth_role')             => 'pe_compiler',
            peadm::oid('peadm_availability_group') => 'A',
            peadm::oid('peadm_legacy_compiler')    => 'true',
          },
        )
      },
      background('modify-compilers-b-certs') || {
        run_plan('peadm::modify_certificate', $legacy_compiler_b_targets,
          primary_host   => $primary_target,
          add_extensions => {
            peadm::oid('pp_auth_role')             => 'pe_compiler',
            peadm::oid('peadm_availability_group') => 'B',
            peadm::oid('peadm_legacy_compiler')    => 'true',
          },
        )
      },
    ]

    # Wait for all the cert modification jobs to complete
    wait($background_cert_jobs)
  }

  peadm::plan_step('convert-node-groups') || {
    # Create the necessary node groups in the console, unless the PE version is
    # too old to support it pre-upgrade. In that circumstance, we trust that
    # the existing groups are correct enough to function until the upgrade is
    # performed.
    if (versioncmp($pe_version, '2019.7.0') >= 0) {
      $rules = run_task('peadm::get_group_rules', $primary_target).first.value['_output']
      $rules_formatted = stdlib::to_json_pretty(parsejson($rules))
      out::message("WARNING: The following existing rules on the PE Infrastructure Agent group will be overwritten with default values:\n ${rules_formatted}")

      apply($primary_target) {
        class { 'peadm::setup::node_manager_yaml':
          primary_host => $primary_target.peadm::certname(),
        }

        class { 'peadm::setup::node_manager':
          primary_host                     => $primary_target.peadm::certname(),
          server_a_host                    => $primary_target.peadm::certname(),
          server_b_host                    => $replica_target.peadm::certname(),
          postgresql_a_host                => $primary_postgresql_target.peadm::certname(),
          postgresql_b_host                => $replica_postgresql_target.peadm::certname(),
          compiler_pool_address            => $compiler_pool_address,
          internal_compiler_a_pool_address => $internal_compiler_a_pool_address,
          internal_compiler_b_pool_address => $internal_compiler_b_pool_address,
          require                          => Class['peadm::setup::node_manager_yaml'],
        }

        include peadm::setup::convert_node_manager
      }
    }
    else {
# lint:ignore:strict_indent
      out::message(@("EOL"/L))
        NOTICE: Node groups not created/updated as part of convert because PE \
        version is too old to support them. Node groups will be updated when \
        the peadm::upgrade plan is run.
        | EOL
# lint:endignore
    }
  }

  peadm::plan_step('finalize') || {
    # Run Puppet on all targets to ensure catalogs and exported resources fully
    # up-to-date. Run on primary first in case puppet server restarts, 'cause
    # that would cause the runs to fail on all the rest.
    run_task('peadm::puppet_runonce', $primary_target)
    run_task('peadm::puppet_runonce', $all_targets - $primary_target)

    # Restart cluster compiler services that are likely not restarted by our
    # final Puppet run to increase chance everything is functional upon plan
    # completion
    if $legacy_compiler_targets {
      run_command('systemctl restart pe-puppetserver.service', $legacy_compiler_targets)
    }

    # PuppetDB is only found on modern compilers, not legacy ones
    if $compiler_targets {
      run_command('systemctl restart pe-puppetserver.service pe-puppetdb.service', $compiler_targets)
    }

    # Run puppet on all targets again to ensure everything is fully up-to-date
    run_task('peadm::puppet_runonce', $all_targets)
  }

  if $legacy_compilers {
# lint:ignore:strict_indent
    $warning_msg = run_task('peadm::check_legacy_compilers', $primary_host, legacy_compilers => $legacy_compilers.join(',') ).first.message
        if $warning_msg.length > 0 {
      out::message(@("WARN"/L))
      WARNING: ${warning_msg}
      | WARN
    }
# lint:endignore
  }

  return("Conversion to peadm Puppet Enterprise ${arch['architecture']} completed.")
}