Skip to content

Commit 9e32967

Browse files
loic-roux-404loic-roux-404
committed
feat: tailscale correct node destroy
1 parent c53aad0 commit 9e32967

26 files changed

+448
-218
lines changed

‎.github/workflows/release-nixos.yml

-3
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,8 @@ on:
66
paths:
77
- 'nixos/**.nix'
88
- 'nixos-options/**.nix'
9-
- 'nixos-generators/**.nix'
109
- flake.nix
1110
- flake.lock
12-
- '!**.md'
13-
- 'packer/**'
1411
- .github/workflows/release-nixos.yml
1512

1613
permissions:

‎nixos/configuration.nix

+9-8
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,6 @@ let
1414
url = "https://github.com/cert-manager/cert-manager/releases/download/v1.14.4/cert-manager.crds.yaml";
1515
sha256 = "060bn3gvrr5jphaig1g195prip5rn0x1s7qrp09q47719fgc6636";
1616
};
17-
manifests = [{
18-
file = certManagerCrds;
19-
toWait = "crd/certificates.cert-manager.io";
20-
namespace = "";
21-
condition = "condition=established";
22-
}];
2317
in {
2418

2519
fileSystems."/" = {
@@ -65,7 +59,7 @@ in {
6559
tailscale = {
6660
enable = true;
6761
openFirewall = true;
68-
extraUpFlags = ["--ssh"];
62+
extraUpFlags = ["--ssh" "--accept-dns"];
6963
extraDaemonFlags = tailscale.baseDaemonExtraArgs;
7064
permitCertUid = user.name;
7165
};
@@ -99,13 +93,20 @@ in {
9993
};
10094

10195
system.activationScripts.k3sCerts.text = (pkgs.callPackage ./install-k3s-manifest.nix {
102-
inherit lib pkgs manifests;
96+
inherit pkgs;
97+
manifest = {
98+
file = certManagerCrds;
99+
toWait = "crd/certificates.cert-manager.io";
100+
namespace = "";
101+
condition = "condition=established";
102+
};
103103
}).script;
104104

105105
environment = {
106106
shells = [ pkgs.bashInteractive ];
107107
variables = {
108108
EDITOR = "vim";
109+
SYSTEMD_EDITOR = "vim";
109110
PAGER = "less -FirSwX";
110111
};
111112
systemPackages = with pkgs; [

‎nixos/contabo.nix

+37-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{ ... }:
1+
{ pkgs, ... }:
22
{
33

44
boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "virtio_pci" "virtio_scsi" "sd_mod" ];
@@ -8,6 +8,42 @@
88

99
swapDevices = [ ];
1010

11+
security.sudo.wheelNeedsPassword = true;
12+
security.sudo = {
13+
enable = true;
14+
extraRules = [{
15+
commands = map (cmd: {
16+
command = cmd;
17+
options = [ "NOPASSWD" ];
18+
}) [
19+
"${pkgs.systemd}/bin/systemctl status"
20+
"${pkgs.systemd}/bin/systemctl show"
21+
"${pkgs.systemd}/bin/systemctl list-units"
22+
"${pkgs.systemd}/bin/systemctl list-machines"
23+
"${pkgs.systemd}/bin/systemctl list-jobs"
24+
"${pkgs.systemd}/bin/systemctl is-system-running"
25+
"${pkgs.systemd}/bin/journalctl"
26+
"${pkgs.k3s}/bin/kubectl get"
27+
"${pkgs.k3s}/bin/kubectl describe"
28+
"${pkgs.k3s}/bin/kubectl explain"
29+
"${pkgs.k3s}/bin/kubectl logs"
30+
"${pkgs.k3s}/bin/kubectl diff"
31+
"${pkgs.k3s}/bin/kubectl events"
32+
"${pkgs.k3s}/bin/kubectl wait"
33+
"${pkgs.k3s}/bin/kubectl api-resources"
34+
"${pkgs.k3s}/bin/kubectl version"
35+
"${pkgs.vim}/bin/vim"
36+
"${pkgs.less}/bin/less"
37+
"${pkgs.coreutils}/bin/tail"
38+
"${pkgs.coreutils}/bin/grep"
39+
"${pkgs.nettools}/bin/ifconfig"
40+
"${pkgs.iproute2}/bin/ip"
41+
"${pkgs.iptables}/bin/iptables"
42+
];
43+
groups = [ "wheel" ];
44+
}];
45+
};
46+
1147
k3s-paas.dns.name = "404-tools.xyz";
1248
k3s-paas.certs = [];
1349
}

‎nixos/install-k3s-manifest.nix

+11-12
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,15 @@
11
{
2-
lib,
32
pkgs,
4-
manifests ? []
3+
manifest,
4+
...
55
} :
6-
{
7-
script = "mkdir -p /var/lib/rancher/k3s/server/manifests;" +
8-
lib.strings.concatMapStrings (manifest:
9-
with manifest;
10-
let namespaceExpr = if namespace != "" then "-n ${namespace}" else ""; in
11-
''
12-
cp -fp ${file} /var/lib/rancher/k3s/server/manifests;
13-
${pkgs.k3s}/bin/kubectl wait --for='${condition}' ${toWait} ${namespaceExpr} --timeout=2m;
14-
''
15-
) manifests;
6+
with manifest;
7+
let namespaceExpr = if namespace != "" then "-n ${namespace}" else "";
8+
in {
9+
script = ''
10+
mkdir -p /var/lib/rancher/k3s/server/manifests;
11+
cp -fp ${file} /var/lib/rancher/k3s/server/manifests;
12+
sleep 30;
13+
${pkgs.k3s}/bin/kubectl wait --for='${condition}' ${toWait} ${namespaceExpr} --timeout=2m;
14+
'';
1615
}

‎nixos/tailscale-deploy.nix

+37-33
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,13 @@
1-
{ config, pkgs, lib, ... }:
1+
{ config, pkgs, ... }:
22

3-
let manifests = [
4-
{
5-
file = config.sops.templates."tailscale.yaml".path;
6-
toWait = "deployment.apps/operator";
7-
namespace = "tailscale";
8-
condition = "condition=Available";
9-
}
10-
{
11-
file = pkgs.writeText "tailscale-namespace.yaml" ''
12-
apiVersion: v1
13-
kind: Namespace
14-
metadata:
15-
name: tailscale
16-
'';
17-
condition = "jsonpath={.status.phase}=Active";
18-
toWait = "namespace/tailscale";
19-
namespace = "";
20-
}
21-
];
22-
in {
23-
services.tailscale.authKeyFile = config.sops.secrets.tailscale.path;
24-
services.tailscale.extraUpFlags = ["--ssh" "--hostname=${config.networking.hostName}"];
3+
{
4+
services.tailscale.authKeyFile = config.sops.secrets.tailscaleNodeKey.path;
5+
services.tailscale.extraUpFlags = ["--ssh" "--accept-dns" ];
256

26-
system.activationScripts.tailscaleOperator.deps = [ "renderSecrets" ];
27-
system.activationScripts.tailscaleOperator.text = (pkgs.callPackage ./install-k3s-manifest.nix {
28-
inherit lib pkgs manifests;
29-
}).script;
30-
31-
sops.secrets.tailscale = {};
32-
sops.secrets.tailscale_oauth_client_id = {};
33-
sops.secrets.tailscale_oauth_client_secret = {};
7+
sops.secrets.tailscaleNodeKey = {};
8+
sops.secrets.tailscaleNodeHostname = {};
9+
sops.secrets.tailscaleOauthClientId = {};
10+
sops.secrets.tailscaleOauthClientSecret = {};
3411

3512
sops.templates."tailscale.yaml".content = ''
3613
apiVersion: helm.cattle.io/v1
@@ -43,12 +20,39 @@ in {
4320
chart: tailscale-operator
4421
targetNamespace: tailscale
4522
valuesContent: |
23+
operatorConfig:
24+
hostname: "k8s-operator-${config.sops.placeholder.tailscaleNodeHostname}"
4625
oauth:
47-
clientId: ${config.sops.placeholder.tailscale_oauth_client_id}
48-
clientSecret: ${config.sops.placeholder.tailscale_oauth_client_secret}
26+
clientId: ${config.sops.placeholder.tailscaleOauthClientId}
27+
clientSecret: ${config.sops.placeholder.tailscaleOauthClientSecret}
4928
apiServerProxyConfig:
5029
mode: "true"
5130
waitForJobs: true
5231
waitForHelm: true
5332
'';
33+
34+
system.activationScripts.tailscaleNamespace.text = (pkgs.callPackage ./install-k3s-manifest.nix {
35+
inherit pkgs;
36+
manifest = {
37+
file = pkgs.writeText "tailscale-namespace.yaml" ''
38+
apiVersion: v1
39+
kind: Namespace
40+
metadata:
41+
name: tailscale
42+
'';
43+
condition = "jsonpath={.status.phase}=Active";
44+
toWait = "namespace/tailscale";
45+
namespace = "";
46+
};
47+
}).script;
48+
system.activationScripts.tailscaleOperator.deps = [ "renderSecrets" "tailscaleNamespace" ];
49+
system.activationScripts.tailscaleOperator.text = (pkgs.callPackage ./install-k3s-manifest.nix {
50+
inherit pkgs;
51+
manifest = {
52+
file = config.sops.templates."tailscale.yaml".path;
53+
toWait = "deployment.apps/operator";
54+
namespace = "tailscale";
55+
condition = "condition=Available";
56+
};
57+
}).script;
5458
}

‎tf-modules-cloud/contabo/main.tf

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
resource "contabo_secret" "k3s_paas_master_trusted_key" {
22
name = "k3s_paas_master_trusted_key"
33
type = "ssh"
4-
value = var.ssh_connection.public_key
4+
value = file(var.ssh_connection.public_key)
55
}
66

77
resource "contabo_image" "k3s_paas_master_image" {
@@ -23,15 +23,15 @@ resource "contabo_instance" "k3s_paas_master" {
2323
ssh_keys = [contabo_secret.k3s_paas_master_trusted_key.id]
2424
}
2525

26-
output "name" {
26+
output "node_hostname" {
2727
depends_on = [ contabo_instance.k3s_paas_master ]
2828
value = contabo_instance.k3s_paas_master.name
2929
}
3030

31-
output "ip" {
31+
output "node_ip" {
3232
value = data.contabo_instance.k3s_paas_master.ip_config[0].v4[0].ip
3333
}
3434

35-
output "id" {
35+
output "node_id" {
3636
value = contabo_instance.k3s_paas_master.id
3737
}

‎tf-modules-cloud/contabo/variables.tf

-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ variable "image_url_format" {
1414
variable "ssh_connection" {
1515
type = object({
1616
user = string
17-
password = string
1817
public_key = string
1918
private_key = string
2019
})

‎tf-modules-cloud/tailscale/delete-node-devices.sh

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/usr/bin/env bash
2+
3+
ENDPOINT=${ENDPOINT:-https://api.tailscale.com}
4+
API_KEY=$(curl -s -d "client_id=$OAUTH_CLIENT_ID" -d "client_secret=$OAUTH_CLIENT_SECRET" \
5+
"${ENDPOINT}/api/v2/oauth/token" | jq -r '.access_token')
6+
NODE_HOSTNAMES=${NODE_HOSTNAMES:-}
7+
8+
IFS=',' read -ra ADDR <<< "$NODE_HOSTNAMES"
9+
for NODE_HOSTNAME in "${ADDR[@]}"; do
10+
11+
curl -s "${ENDPOINT}/api/v2/tailnet/$TAILNET/devices" -u "$API_KEY:" | jq -r '.devices[] | "\(.id) \(.name)"' |
12+
while read -r id name; do
13+
if [[ $name = *"$NODE_HOSTNAME.$TAILNET"* ]]
14+
then
15+
echo "$name matching $NODE_HOSTNAME.$TAILNET - getting rid of $id"
16+
curl -s -X DELETE "${ENDPOINT}/api/v2/device/$id" -u "$API_KEY:"
17+
else
18+
echo "$name not matching $NODE_HOSTNAME.$TAILNET, keeping it"
19+
fi
20+
done
21+
done

‎tf-modules-cloud/tailscale/main.tf

+42-6
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,9 @@ resource "tailscale_acl" "as_json" {
3434
users : ["autogroup:nonroot"]
3535
},
3636
{
37-
action: "accept",
38-
src: ["autogroup:member"],
39-
dst: ["tag:k8s-operator"],
37+
action : "accept",
38+
src : ["autogroup:member"],
39+
dst : ["tag:k8s-operator"],
4040
users : ["autogroup:nonroot"]
4141
},
4242
],
@@ -69,17 +69,53 @@ resource "tailscale_dns_preferences" "sample_preferences" {
6969
magic_dns = true
7070
}
7171

72+
resource "terraform_data" "node_changed" {
73+
triggers_replace = [var.node_id]
74+
}
75+
7276
resource "tailscale_tailnet_key" "k3s_paas_node" {
7377
depends_on = [tailscale_acl.as_json]
7478
reusable = true
75-
ephemeral = true
79+
ephemeral = false
7680
expiry = 3600
7781
recreate_if_invalid = "always"
7882
preauthorized = true
7983
description = "VM instance key"
8084
tags = ["tag:all"]
8185
}
8286

83-
output "key" {
84-
value = tailscale_tailnet_key.k3s_paas_node.key
87+
resource "terraform_data" "destroy_node" {
88+
input = {
89+
TAILNET = var.tailscale_tailnet
90+
OAUTH_CLIENT_ID = var.tailscale_oauth_client.id
91+
OAUTH_CLIENT_SECRET = var.tailscale_oauth_client.secret
92+
NODE_HOSTNAMES = join(",", [
93+
var.node_hostname,
94+
"k8s-operator-${var.node_hostname}"
95+
])
96+
}
97+
98+
provisioner "local-exec" {
99+
when = destroy
100+
environment = self.input
101+
on_failure = fail
102+
command = "${path.module}/delete-node-devices.sh"
103+
}
104+
}
105+
106+
output "node_id" {
107+
value = var.node_id
108+
}
109+
110+
output "node_ip" {
111+
value = var.node_ip
112+
}
113+
114+
output "config" {
115+
depends_on = [tailscale_tailnet_key.k3s_paas_node]
116+
value = {
117+
node_hostname = var.node_hostname
118+
node_fqdn = "${var.node_hostname}.${var.tailscale_tailnet}"
119+
node_key = tailscale_tailnet_key.k3s_paas_node.key
120+
}
85121
}

‎tf-modules-cloud/tailscale/variables.tf

+20
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,23 @@ variable "trusted_ssh_user" {
99
variable "tailscale_tailnet" {
1010
type = string
1111
}
12+
13+
variable "node_hostname" {
14+
type = string
15+
}
16+
17+
variable "node_ip" {
18+
type = string
19+
}
20+
21+
variable "node_id" {
22+
type = string
23+
}
24+
25+
variable "tailscale_oauth_client" {
26+
sensitive = true
27+
type = object({
28+
id = string
29+
secret = string
30+
})
31+
}

‎tf-modules-k8s/waypoint-config/main.tf

+2-2
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ locals {
3232
])
3333
}
3434

35-
resource "null_resource" "setup_oidc" {
36-
triggers = {
35+
resource "terraform_data" "setup_oidc" {
36+
triggers_replace = {
3737
login_cmd = local.login_cmd
3838
oidc_setup_cmd = local.oidc_setup_cmd
3939
}

‎tf-modules-nix/deploy/key-to-age.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,6 @@
22

33
eval "$(jq -r '@sh "key=\(.key) args=\(.args)"')"
44

5-
OUTPUT=$(echo "$key" | ssh-to-age "${args:-}")
5+
OUTPUT=$(ssh-to-age "${args:-}" < "$key")
66

77
jq -n --arg output "$OUTPUT" '{"key": $output}'

0 commit comments

Comments
 (0)