Puppet Class: spark

Inherits:
::spark::params
Defined in:
manifests/init.pp

Overview

Class: spark

Main configuration class for CESNET Apache Spark puppet module.

Parameters:

  • alternatives (Any) (defaults to: '::default')
  • hdfs_hostname (Any) (defaults to: undef)
  • master_hostname (Any) (defaults to: undef)
  • master_port (Any) (defaults to: $params::master_port)
  • master_ui_port (Any) (defaults to: $params::master_ui_port)
  • historyserver_hostname (Any) (defaults to: undef)
  • historyserver_port (Any) (defaults to: $params::historyserver_port)
  • worker_port (Any) (defaults to: $params::worker_port)
  • worker_ui_port (Any) (defaults to: $params::worker_ui_port)
  • environment (Any) (defaults to: undef)
  • properties (Any) (defaults to: undef)
  • realm (Any) (defaults to: undef)
  • hive_enable (Any) (defaults to: true)
  • jar_enable (Any) (defaults to: false)
  • yarn_enable (Any) (defaults to: true) 


5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'manifests/init.pp', line 5

class spark (
  $alternatives = '::default',
  $hdfs_hostname = undef,
  $master_hostname = undef,
  $master_port = $params::master_port,
  $master_ui_port = $params::master_ui_port,
  $historyserver_hostname = undef,
  $historyserver_port = $params::historyserver_port,
  $worker_port = $params::worker_port,
  $worker_ui_port = $params::worker_ui_port,
  $environment = undef,
  $properties = undef,
  $realm = undef,
  $hive_enable = true,
  $jar_enable = false,
  $yarn_enable = true,
) inherits ::spark::params {
  include ::stdlib

  if $alternatives {
    validate_string($alternatives)
  }
  validate_bool($jar_enable)
  if $jar_enable and !$hdfs_hostname {
    warn('$hdfs_hostname parameter needed, when remote copied jar enabled')
  }

  if $historyserver_hostname {
    $hs_properties = {
      # must be with 'http://' to proper redirection from Hadoop with security
      'spark.yarn.historyServer.address' => "http://${historyserver_hostname}:${historyserver_port}",
    }
  }
  if $historyserver_hostname == $::fqdn {
    $hs_daemon_properties = {
      'spark.history.ui.port' => $historyserver_port,
    }
  } else {
    $hs_daemon_properties = {}
  }
  if $realm and $realm != '' {
    if $historyserver_hostname == $::fqdn {
      $security_properties = {
        'spark.history.kerberos.enabled' => true,
        'spark.history.kerberos.keytab' => '/etc/security/keytab/spark.service.keytab',
        'spark.history.kerberos.principal' => "spark/${::fqdn}@${realm}",
      }
    } else {
      $security_properties = {}
    }
  } else {
    $security_properties = {}
  }

  $_properties = merge($hs_properties, $hs_daemon_properties, $security_properties, $properties)
}