terraform rightly describes itself as a provisioning tool, and not a config management tool. So this idea of using terraform to manage the config for a server didn't really work out.
use provisioner "file"s on null_resources to transfer the config files
necessary to run nomad.
You have to do it right the first time,
otherwise it's a manual taint of the resource and recreating it.
Also it doesn't create the directories for a specified path,
necessitating provisioner "remote-exec" just to create the dirs.
I should (have to?) reboot the server after everything is done, but terraform doesn't make doing that + exiting properly / validating easy. So I copped out and just did that part by hand.
This file creates a self signed CA
and a set of client certs output to a local dir
so local nomad cli commands can authenticate.
1##################################################
2# ca certs
3##################################################
4resource "tls_private_key" "ca" {
5 algorithm = "ECDSA"
6 ecdsa_curve = "P384"
7}
8
9resource "tls_self_signed_cert" "ca" {
10 key_algorithm = tls_private_key.ca.algorithm
11 private_key_pem = tls_private_key.ca.private_key_pem
12 is_ca_certificate = true
13
14 validity_period_hours = 24 * 365 * 10
15
16 allowed_uses = [
17 "cert_signing",
18 ]
19
20 subject {
21 common_name = "medea.seankhliao.com"
22 organization = "medea / seankhliao"
23 }
24}
25resource "local_file" "ca_cert" {
26 content = tls_self_signed_cert.ca.cert_pem
27 filename = "${path.root}/out/ca.crt"
28}
29
30
31##################################################
32# client cert
33##################################################
34resource "tls_private_key" "nomad_client" {
35 algorithm = "ECDSA"
36 ecdsa_curve = "P384"
37}
38resource "local_file" "nomad_client_key" {
39 content = tls_private_key.nomad_client.private_key_pem
40 filename = "${path.root}/out/client.key"
41}
42
43resource "tls_cert_request" "nomad_client" {
44 key_algorithm = "ECDSA"
45 private_key_pem = tls_private_key.nomad_client.private_key_pem
46
47 subject {
48 common_name = "eevee.seankhliao.com"
49 organization = "medea / seankhliao"
50 }
51}
52resource "tls_locally_signed_cert" "nomad_client" {
53 ca_key_algorithm = tls_private_key.ca.algorithm
54 ca_private_key_pem = tls_private_key.ca.private_key_pem
55 ca_cert_pem = tls_self_signed_cert.ca.cert_pem
56 cert_request_pem = tls_cert_request.nomad_client.cert_request_pem
57
58 validity_period_hours = 24 * 365
59
60 allowed_uses = [
61 "digital_signature",
62 "key_encipherment",
63 "client_auth",
64 ]
65}
66resource "local_file" "nomad_client_cert" {
67 content = tls_locally_signed_cert.nomad_client.cert_pem
68 filename = "${path.root}/out/client.crt"
69}
medea is the name of my server.
This file manages the basic setup i need from a fresh image.
1resource "null_resource" "medea" {
2 connection {
3 host = "medea.seankhliao.com"
4 private_key = file(pathexpand("~/.ssh/id_ed25519"))
5 agent = false
6 }
7
8 provisioner "remote-exec" {
9 inline = [
10 "rm /etc/sysctl.d/* || true",
11 "rm /etc/ssh/ssh_host_{dsa,rsa,ecdsa}_key* || true",
12 "pacman -Rns --noconfirm btrfs-progs gptfdisk haveged xfsprogs wget vim net-tools cronie || true",
13 "pacman -Syu --noconfirm neovim zip unzip",
14 "systemctl enable --now systemd-timesyncd",
15 ]
16 }
17 provisioner "file" {
18 destination = "/root/.ssh/authorized_keys"
19 content = <<-EOT
20 ${file(pathexpand("~/.ssh/id_ed25519.pub"))}
21 ${file(pathexpand("~/.ssh/id_ed25519_sk.pub"))}
22 ${file(pathexpand("~/.ssh/id_ecdsa_sk.pub"))}
23 EOT
24 }
25 provisioner "file" {
26 destination = "/etc/sysctl.d/30-ipforward.conf"
27 content = <<-EOT
28 net.ipv4.ip_forward=1
29 net.ipv4.conf.lxc*.rp_filter=0
30 net.ipv6.conf.default.forwarding=1
31 net.ipv6.conf.all.forwarding=1
32 EOT
33 }
34 provisioner "file" {
35 destination = "/etc/modules-load.d/br_netfilter.conf"
36 content = "br_netfilter"
37 }
38 provisioner "file" {
39 destination = "/etc/systemd/network/40-wg0.netdev"
40 content = <<-EOT
41 # WireGuard
42
43 [NetDev]
44 Name = wg0
45 Kind = wireguard
46
47 [WireGuard]
48 PrivateKey = xxxx
49 # PublicKey = lombY0b15giOmoM9t0xBi+UgVkZDoOKDaEV9+ONwH1U=
50 ListenPort = 51820
51
52 # eevee
53 [WireGuardPeer]
54 PublicKey = YvSLDXl3NX1ySTX2C8D72+fCVBcqSs+fmAX3uySCDAQ=
55 AllowedIPs = 192.168.100.13/32
56
57 # pixel 3
58 [WireGuardPeer]
59 PublicKey = 3xpGlOORQb9/yg545KX+odrup3YaslxO9ie+ztJ3Y3E=
60 AllowedIPs = 192.168.100.3/32
61
62 # arch
63 [WireGuardPeer]
64 PublicKey = Lr17jGvc7uwjn9LNRR+IkCkjuP8nkHTOMHbVV+onMn0=
65 AllowedIPs = 192.168.100.1/24
66 Endpoint = yyyy
67 EOT
68 }
69 provisioner "file" {
70 destination = "/etc/systemd/network/41-wg0.network"
71 content = <<-EOT
72 [Match]
73 Name = wg0
74
75 [Network]
76 Address = 192.168.100.2/24
77 EOT
78 }
79}
This file provisions the server(+client) certs for nomad
(it needs client to talk to itself),
downloads nomad and runs it with systemd.
1##################################################
2# nomad server cert
3##################################################
4resource "tls_private_key" "nomad_server" {
5 algorithm = "ECDSA"
6 ecdsa_curve = "P384"
7}
8
9resource "tls_cert_request" "nomad_server" {
10 key_algorithm = "ECDSA"
11 private_key_pem = tls_private_key.nomad_server.private_key_pem
12
13 dns_names = [
14 "localhost",
15 "server.global.nomad",
16 "medea.seankhliao.com",
17 ]
18
19 ip_addresses = [
20 "127.0.0.1",
21 "192.168.100.2",
22 "65.21.73.144",
23 ]
24
25 subject {
26 common_name = "medea.seankhliao.com"
27 organization = "medea / seankhliao"
28 }
29}
30resource "tls_locally_signed_cert" "nomad_server" {
31 ca_key_algorithm = tls_private_key.ca.algorithm
32 ca_private_key_pem = tls_private_key.ca.private_key_pem
33 ca_cert_pem = tls_self_signed_cert.ca.cert_pem
34 cert_request_pem = tls_cert_request.nomad_server.cert_request_pem
35
36 validity_period_hours = 24 * 365
37
38 allowed_uses = [
39 "digital_signature",
40 "key_encipherment",
41 "server_auth",
42 "client_auth",
43 ]
44}
45
46##################################################
47# nomad
48##################################################
49resource "null_resource" "medea_nomad" {
50 depends_on = [
51 null_resource.medea,
52 ]
53
54 connection {
55 host = "medea.seankhliao.com"
56 private_key = file(pathexpand("~/.ssh/id_ed25519"))
57 agent = false
58 }
59
60 provisioner "file" {
61 destination = "/etc/systemd/system/nomad.service"
62 content = <<-EOT
63 [Unit]
64 Description=nomad server
65 Documentation=https://www.nomadproject.io/docs/agent/
66 Requires=network-online.target
67 After=network-online.target
68
69 [Service]
70 Restart=on-failure
71 ExecStart=/usr/local/bin/nomad agent -config /etc/nomad/server.hcl
72 ExecReload=/usr/bin/kill -HUP $MAINPID
73 KillSignal=SIGINT
74
75 [Install]
76 WantedBy=multi-user.target
77 EOT
78 }
79 provisioner "remote-exec" {
80 inline = [
81 "curl -Lo /tmp/nomad.zip https://releases.hashicorp.com/nomad/1.1.0/nomad_1.1.0_linux_amd64.zip",
82 "unzip -o /tmp/nomad.zip nomad -d /usr/local/bin",
83 "systemctl daemon-reload",
84 "systemctl enable nomad",
85 "rm -rf /etc/nomad || true",
86 "mkdir -p /etc/nomad",
87 ]
88 }
89 provisioner "file" {
90 destination = "/etc/nomad/ca.crt"
91 content = tls_self_signed_cert.ca.cert_pem
92 }
93 provisioner "file" {
94 destination = "/etc/nomad/server.key"
95 content = tls_private_key.nomad_server.private_key_pem
96 }
97 provisioner "file" {
98 destination = "/etc/nomad/server.crt"
99 content = tls_locally_signed_cert.nomad_server.cert_pem
100 }
101 provisioner "file" {
102 destination = "/etc/nomad/server.hcl"
103 content = <<-EOT
104 bind_addr = "0.0.0.0"
105 data_dir = "/var/lib/nomad"
106
107 tls {
108 http = true
109 rpc = true
110
111 ca_file = "/etc/nomad/ca.crt"
112 cert_file = "/etc/nomad/server.crt"
113 key_file = "/etc/nomad/server.key"
114
115 verify_server_hostname = true
116 verify_https_client = true
117 }
118
119 leave_on_interrupt = true
120 leave_on_terminate = true
121
122 disable_update_check = true
123
124 server {
125 enabled = true
126 bootstrap_expect = 1
127 }
128
129 client {
130 enabled = true
131 }
132
133 log_level = "INFO"
134 EOT
135 }
136}
Similar, but for consul. The other option is to run it inside nomad as a system job, which may actually work better, since this is janky. Once nomad has talked to consul, it doesn't like being disconnected.
1##################################################
2# consul server cert
3##################################################
4resource "tls_private_key" "consul_server" {
5 algorithm = "ECDSA"
6 ecdsa_curve = "P384"
7}
8
9resource "tls_cert_request" "consul_server" {
10 key_algorithm = "ECDSA"
11 private_key_pem = tls_private_key.consul_server.private_key_pem
12
13 dns_names = [
14 "localhost",
15 "server.dc1.consul",
16 "medea.seankhliao.com",
17 ]
18
19 ip_addresses = [
20 "127.0.0.1",
21 "192.168.100.2",
22 "65.21.73.144",
23 ]
24
25 subject {
26 common_name = "medea.seankhliao.com"
27 organization = "medea / seankhliao"
28 }
29}
30resource "tls_locally_signed_cert" "consul_server" {
31 ca_key_algorithm = tls_private_key.ca.algorithm
32 ca_private_key_pem = tls_private_key.ca.private_key_pem
33 ca_cert_pem = tls_self_signed_cert.ca.cert_pem
34 cert_request_pem = tls_cert_request.consul_server.cert_request_pem
35
36 validity_period_hours = 24 * 365
37
38 allowed_uses = [
39 "digital_signature",
40 "key_encipherment",
41 "server_auth",
42 "client_auth",
43 ]
44}
45
46##################################################
47# consul
48##################################################
49resource "null_resource" "medea_consul" {
50 depends_on = [
51 null_resource.medea,
52 ]
53
54 connection {
55 host = "medea.seankhliao.com"
56 private_key = file(pathexpand("~/.ssh/id_ed25519"))
57 agent = false
58 }
59
60 provisioner "file" {
61 destination = "/etc/systemd/system/consul.service"
62 content = <<-EOT
63 [Unit]
64 Description=Consul Agent
65 Documentation=https://consul.io/docs/
66 Requires=network-online.target
67 After=network-online.target
68
69 [Service]
70 Restart=on-failure
71 ExecStart=/usr/local/bin/consul agent -config-file /etc/consul/server.hcl
72 ExecReload=/usr/bin/kill -HUP $MAINPID
73 KillSignal=SIGINT
74
75 [Install]
76 WantedBy=multi-user.target
77 EOT
78 }
79 provisioner "remote-exec" {
80 inline = [
81 "curl -Lo /tmp/consul.zip https://releases.hashicorp.com/consul/1.9.5/consul_1.9.5_linux_amd64.zip",
82 "unzip -o /tmp/consul.zip consul -d /usr/local/bin",
83 "systemctl daemon-reload",
84 "systemctl enable consul",
85 "rm -rf /etc/consul || true",
86 "mkdir -p /etc/consul",
87 ]
88 }
89 provisioner "file" {
90 destination = "/etc/consul/ca.crt"
91 content = tls_self_signed_cert.ca.cert_pem
92 }
93 provisioner "file" {
94 destination = "/etc/consul/server.key"
95 content = tls_private_key.consul_server.private_key_pem
96 }
97 provisioner "file" {
98 destination = "/etc/consul/server.crt"
99 content = tls_locally_signed_cert.consul_server.cert_pem
100 }
101 provisioner "file" {
102 destination = "/etc/consul/server.hcl"
103 content = <<-EOT
104 bind_addr = "0.0.0.0"
105 data_dir = "/var/lib/consul"
106 ports {
107 http = -1
108 https = 8501
109 grpc = 8502
110 }
111
112 ca_file = "/etc/consul/ca.crt"
113 cert_file = "/etc/consul/server.crt"
114 key_file = "/etc/consul/server.key"
115
116 verify_incoming = true
117 verify_server_hostname = true
118
119 leave_on_terminate = true
120
121 disable_update_check = true
122
123 server = true
124 bootstrap_expect = 1
125
126 client_addr = "0.0.0.0"
127
128 ui_config {
129 enabled = true
130 }
131
132 log_level = "INFO"
133 EOT
134 }
135}