Skip to content

Commit

Permalink
Revert "refactor: move custom sysctls to cse" (#2863) (#2864)
Browse files Browse the repository at this point in the history 
This reverts commit 92060d0.
  • Loading branch information
@alexeldeib
alexeldeib authored Mar 13, 2023
1 parent a6dae72 commit 57ca948
Show file tree
Hide file tree
Showing 149 changed files with 1,235 additions and 416 deletions.
1 change: 0 additions & 1 deletion parts/linux/cloud-init/artifacts/cse_cmd.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,6 @@ CUSTOM_SEARCH_REALM_PASSWORD="{{GetSearchDomainRealmPassword}}"
MESSAGE_OF_THE_DAY="{{GetMessageOfTheDay}}"
HAS_KUBELET_DISK_TYPE="{{HasKubeletDiskType}}"
NEEDS_CGROUPV2="{{Is2204VHD}}"
SYSCTL_CONTENT="{{GetSysctlContent}}"
TLS_BOOTSTRAP_TOKEN="{{GetTLSBootstrapTokenForKubeConfig}}"
KUBELET_FLAGS="{{GetKubeletConfigKeyVals}}"
NETWORK_POLICY="{{GetParameter "networkPolicy"}}"
Expand Down
5 changes: 1 addition & 4 deletions parts/linux/cloud-init/artifacts/cse_config.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -442,10 +442,7 @@ EOF

ensureSysctl() {
SYSCTL_CONFIG_FILE=/etc/sysctl.d/999-sysctl-aks.conf
mkdir -p "$(dirname "${SYSCTL_CONFIG_FILE}")"
touch "${SYSCTL_CONFIG_FILE}"
chmod 0644 "${SYSCTL_CONFIG_FILE}"
echo "${SYSCTL_CONTENT}" | base64 -d > "${SYSCTL_CONFIG_FILE}"
wait_for_file 1200 1 $SYSCTL_CONFIG_FILE || exit $ERR_FILE_WATCH_TIMEOUT
retrycmd_if_failure 24 5 25 sysctl --system
}

Expand Down
110 changes: 110 additions & 0 deletions parts/linux/cloud-init/nodecustomdata.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -395,3 +395,113 @@ write_files:
owner: root
content: !!binary |
{{GetVariableProperty "cloudInitData" "customSearchDomainsScript"}}


- path: /etc/sysctl.d/999-sysctl-aks.conf
permissions: "0644"
owner: root
content: |
# This is a partial workaround to this upstream Kubernetes issue:
# https://github.com/kubernetes/kubernetes/issues/41916#issuecomment-312428731
net.ipv4.tcp_retries2=8
net.core.message_burst=80
net.core.message_cost=40
{{- if GetCustomSysctlConfigByName "NetCoreSomaxconn"}}
net.core.somaxconn={{.CustomLinuxOSConfig.Sysctls.NetCoreSomaxconn}}
{{- else}}
net.core.somaxconn=16384
{{- end}}
{{- if GetCustomSysctlConfigByName "NetIpv4TcpMaxSynBacklog"}}
net.ipv4.tcp_max_syn_backlog={{.CustomLinuxOSConfig.Sysctls.NetIpv4TcpMaxSynBacklog}}
{{- else}}
net.ipv4.tcp_max_syn_backlog=16384
{{- end}}
{{- if GetCustomSysctlConfigByName "NetIpv4NeighDefaultGcThresh1"}}
net.ipv4.neigh.default.gc_thresh1={{.CustomLinuxOSConfig.Sysctls.NetIpv4NeighDefaultGcThresh1}}
{{- else}}
net.ipv4.neigh.default.gc_thresh1=4096
{{- end}}
{{- if GetCustomSysctlConfigByName "NetIpv4NeighDefaultGcThresh2"}}
net.ipv4.neigh.default.gc_thresh2={{.CustomLinuxOSConfig.Sysctls.NetIpv4NeighDefaultGcThresh2}}
{{- else}}
net.ipv4.neigh.default.gc_thresh2=8192
{{- end}}
{{- if GetCustomSysctlConfigByName "NetIpv4NeighDefaultGcThresh3"}}
net.ipv4.neigh.default.gc_thresh3={{.CustomLinuxOSConfig.Sysctls.NetIpv4NeighDefaultGcThresh3}}
{{- else}}
net.ipv4.neigh.default.gc_thresh3=16384
{{- end}}
{{if ShouldConfigCustomSysctl}}
# The following are sysctl configs passed from API
{{- $s:=.CustomLinuxOSConfig.Sysctls}}
{{- if $s.NetCoreNetdevMaxBacklog}}
net.core.netdev_max_backlog={{$s.NetCoreNetdevMaxBacklog}}
{{- end}}
{{- if $s.NetCoreRmemDefault}}
net.core.rmem_default={{$s.NetCoreRmemDefault}}
{{- end}}
{{- if $s.NetCoreRmemMax}}
net.core.rmem_max={{$s.NetCoreRmemMax}}
{{- end}}
{{- if $s.NetCoreWmemDefault}}
net.core.wmem_default={{$s.NetCoreWmemDefault}}
{{- end}}
{{- if $s.NetCoreWmemMax}}
net.core.wmem_max={{$s.NetCoreWmemMax}}
{{- end}}
{{- if $s.NetCoreOptmemMax}}
net.core.optmem_max={{$s.NetCoreOptmemMax}}
{{- end}}
{{- if $s.NetIpv4TcpMaxTwBuckets}}
net.ipv4.tcp_max_tw_buckets={{$s.NetIpv4TcpMaxTwBuckets}}
{{- end}}
{{- if $s.NetIpv4TcpFinTimeout}}
net.ipv4.tcp_fin_timeout={{$s.NetIpv4TcpFinTimeout}}
{{- end}}
{{- if $s.NetIpv4TcpKeepaliveTime}}
net.ipv4.tcp_keepalive_time={{$s.NetIpv4TcpKeepaliveTime}}
{{- end}}
{{- if $s.NetIpv4TcpKeepaliveProbes}}
net.ipv4.tcp_keepalive_probes={{$s.NetIpv4TcpKeepaliveProbes}}
{{- end}}
{{- if $s.NetIpv4TcpkeepaliveIntvl}}
net.ipv4.tcp_keepalive_intvl={{$s.NetIpv4TcpkeepaliveIntvl}}
{{- end}}
{{- if $s.NetIpv4TcpTwReuse}}
net.ipv4.tcp_tw_reuse={{BoolPtrToInt $s.NetIpv4TcpTwReuse}}
{{- end}}
{{- if $s.NetIpv4IpLocalPortRange}}
net.ipv4.ip_local_port_range={{$s.NetIpv4IpLocalPortRange}}
{{- end}}
{{- if $s.NetNetfilterNfConntrackMax}}
net.netfilter.nf_conntrack_max={{$s.NetNetfilterNfConntrackMax}}
{{- end}}
{{- if $s.NetNetfilterNfConntrackBuckets}}
net.netfilter.nf_conntrack_buckets={{$s.NetNetfilterNfConntrackBuckets}}
{{- end}}
{{- if $s.FsInotifyMaxUserWatches}}
fs.inotify.max_user_watches={{$s.FsInotifyMaxUserWatches}}
{{- end}}
{{- if $s.FsFileMax}}
fs.file-max={{$s.FsFileMax}}
{{- end}}
{{- if $s.FsAioMaxNr}}
fs.aio-max-nr={{$s.FsAioMaxNr}}
{{- end}}
{{- if $s.FsNrOpen}}
fs.nr_open={{$s.FsNrOpen}}
{{- end}}
{{- if $s.KernelThreadsMax}}
kernel.threads-max={{$s.KernelThreadsMax}}
{{- end}}
{{- if $s.VMMaxMapCount}}
vm.max_map_count={{$s.VMMaxMapCount}}
{{- end}}
{{- if $s.VMSwappiness}}
vm.swappiness={{$s.VMSwappiness}}
{{- end}}
{{- if $s.VMVfsCachePressure}}
vm.vfs_cache_pressure={{$s.VMVfsCachePressure}}
{{- end}}
{{- end}}
#EOF
118 changes: 0 additions & 118 deletions pkg/agent/baker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -386,9 +386,6 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration
},
"GetCustomSysctlConfigByName": func(fn string) interface{} {
if profile.CustomLinuxOSConfig != nil && profile.CustomLinuxOSConfig.Sysctls != nil {
// TODO(ace): this should be removed.
// yes, enumerating fields of a struct is annoying without reflection.
// that means your api/implementation is probably wrong.
v := reflect.ValueOf(*profile.CustomLinuxOSConfig.Sysctls)
return v.FieldByName(fn).Interface()
}
Expand Down Expand Up @@ -883,14 +880,6 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration
"ShouldDisableSSH": func() bool {
return config.SSHStatus == datamodel.SSHOff
},
"GetSysctlContent": func() string {
sysctlTemplate := template.Must(template.New("sysctl").Parse(sysctlTemplateString))
var b bytes.Buffer
if err := sysctlTemplate.Execute(&b, profile); err != nil {
panic(fmt.Errorf("failed to execute sysctl template: %s", err))
}
return base64.StdEncoding.EncodeToString(b.Bytes())
},
}
}

Expand Down Expand Up @@ -929,113 +918,6 @@ func isMariner(osSku string) bool {
return osSku == datamodel.OSSKUCBLMariner || osSku == datamodel.OSSKUMariner
}

const sysctlTemplateString = `# This is a partial workaround to this upstream Kubernetes issue:
# https://github.com/kubernetes/kubernetes/issues/41916#issuecomment-312428731
net.ipv4.tcp_retries2=8
net.core.message_burst=80
net.core.message_cost=40
{{- if .CustomLinuxOSConfig}}{{ if .CustomLinuxOSConfig.Sysctls}}{{ if .CustomLinuxOSConfig.Sysctls.NetCoreSomaxconn}}
net.core.somaxconn={{.CustomLinuxOSConfig.Sysctls.NetCoreSomaxconn}}
{{end}}{{end}}{{- else}}
net.core.somaxconn=16384
{{- end}}
{{- if .CustomLinuxOSConfig}}{{ if .CustomLinuxOSConfig.Sysctls}}{{ if .CustomLinuxOSConfig.Sysctls.NetIpv4TcpMaxSynBacklog}}
net.ipv4.tcp_max_syn_backlog={{.CustomLinuxOSConfig.Sysctls.NetIpv4TcpMaxSynBacklog}}
{{end}}{{end}}{{- else}}
net.ipv4.tcp_max_syn_backlog=16384
{{- end}}
{{- if .CustomLinuxOSConfig}}{{ if .CustomLinuxOSConfig.Sysctls}}{{ if .CustomLinuxOSConfig.Sysctls.NetIpv4NeighDefaultGcThresh1}}
net.ipv4.neigh.default.gc_thresh1={{.CustomLinuxOSConfig.Sysctls.NetIpv4NeighDefaultGcThresh1}}
{{end}}{{end}}{{- else}}
net.ipv4.neigh.default.gc_thresh1=4096
{{- end}}
{{- if .CustomLinuxOSConfig}}{{ if .CustomLinuxOSConfig.Sysctls}}{{ if .CustomLinuxOSConfig.Sysctls.NetIpv4NeighDefaultGcThresh2}}
net.ipv4.neigh.default.gc_thresh2={{.CustomLinuxOSConfig.Sysctls.NetIpv4NeighDefaultGcThresh2}}
{{end}}{{end}}{{- else}}
net.ipv4.neigh.default.gc_thresh2=8192
{{- end}}
{{- if .CustomLinuxOSConfig}}{{ if .CustomLinuxOSConfig.Sysctls}}{{ if .CustomLinuxOSConfig.Sysctls.NetIpv4NeighDefaultGcThresh3}}
net.ipv4.neigh.default.gc_thresh3={{.CustomLinuxOSConfig.Sysctls.NetIpv4NeighDefaultGcThresh3}}
{{end}}{{end}}{{- else}}
net.ipv4.neigh.default.gc_thresh3=16384
{{- end}}
{{if .CustomLinuxOSConfig}}
{{if .CustomLinuxOSConfig.Sysctls}}
# The following are sysctl configs passed from API
{{- $s:=.CustomLinuxOSConfig.Sysctls}}
{{- if $s.NetCoreNetdevMaxBacklog}}
net.core.netdev_max_backlog={{$s.NetCoreNetdevMaxBacklog}}
{{- end}}
{{- if $s.NetCoreRmemDefault}}
net.core.rmem_default={{$s.NetCoreRmemDefault}}
{{- end}}
{{- if $s.NetCoreRmemMax}}
net.core.rmem_max={{$s.NetCoreRmemMax}}
{{- end}}
{{- if $s.NetCoreWmemDefault}}
net.core.wmem_default={{$s.NetCoreWmemDefault}}
{{- end}}
{{- if $s.NetCoreWmemMax}}
net.core.wmem_max={{$s.NetCoreWmemMax}}
{{- end}}
{{- if $s.NetCoreOptmemMax}}
net.core.optmem_max={{$s.NetCoreOptmemMax}}
{{- end}}
{{- if $s.NetIpv4TcpMaxTwBuckets}}
net.ipv4.tcp_max_tw_buckets={{$s.NetIpv4TcpMaxTwBuckets}}
{{- end}}
{{- if $s.NetIpv4TcpFinTimeout}}
net.ipv4.tcp_fin_timeout={{$s.NetIpv4TcpFinTimeout}}
{{- end}}
{{- if $s.NetIpv4TcpKeepaliveTime}}
net.ipv4.tcp_keepalive_time={{$s.NetIpv4TcpKeepaliveTime}}
{{- end}}
{{- if $s.NetIpv4TcpKeepaliveProbes}}
net.ipv4.tcp_keepalive_probes={{$s.NetIpv4TcpKeepaliveProbes}}
{{- end}}
{{- if $s.NetIpv4TcpkeepaliveIntvl}}
net.ipv4.tcp_keepalive_intvl={{$s.NetIpv4TcpkeepaliveIntvl}}
{{- end}}
{{- if $s.NetIpv4TcpTwReuse}}
net.ipv4.tcp_tw_reuse={{if $s.NetIpv4TcpTwReuse}}1{{else}}0{{end}}
{{- end}}
{{- if $s.NetIpv4IpLocalPortRange}}
net.ipv4.ip_local_port_range={{$s.NetIpv4IpLocalPortRange}}
{{- end}}
{{- if $s.NetNetfilterNfConntrackMax}}
net.netfilter.nf_conntrack_max={{$s.NetNetfilterNfConntrackMax}}
{{- end}}
{{- if $s.NetNetfilterNfConntrackBuckets}}
net.netfilter.nf_conntrack_buckets={{$s.NetNetfilterNfConntrackBuckets}}
{{- end}}
{{- if $s.FsInotifyMaxUserWatches}}
fs.inotify.max_user_watches={{$s.FsInotifyMaxUserWatches}}
{{- end}}
{{- if $s.FsFileMax}}
fs.file-max={{$s.FsFileMax}}
{{- end}}
{{- if $s.FsAioMaxNr}}
fs.aio-max-nr={{$s.FsAioMaxNr}}
{{- end}}
{{- if $s.FsNrOpen}}
fs.nr_open={{$s.FsNrOpen}}
{{- end}}
{{- if $s.KernelThreadsMax}}
kernel.threads-max={{$s.KernelThreadsMax}}
{{- end}}
{{- if $s.VMMaxMapCount}}
vm.max_map_count={{$s.VMMaxMapCount}}
{{- end}}
{{- if $s.VMSwappiness}}
vm.swappiness={{$s.VMSwappiness}}
{{- end}}
{{- if $s.VMVfsCachePressure}}
vm.vfs_cache_pressure={{$s.VMVfsCachePressure}}
{{- end}}
{{- end}}
{{- end}}
`

const kubenetCniTemplate = `
{
"cniVersion": "0.3.1",
Expand Down
Loading

0 comments on commit 57ca948

Please sign in to comment.