Verified Commit 4baee7b8 authored by Caleb Woodbine's avatar Caleb Woodbine
Browse files

Add cluster network booting documentation

parent ab5f36ba
#+NAME: Pi network boot
#+AUTHOR: Caleb Woodbine <caleb@ii.coop>
Setting up a Raspberry Pi boot server in-cluster.
Current limitations as of current version:
- non highly-available and tied to a node
* Pre-setup
#+name: Label a node to allow scheduling of the apps
#+begin_src sh
kubectl label node srv1.ii.nz ii-pi-boot=true
#+end_src
#+name: Create the namespace
#+begin_src sh
kubectl create namespace ii-pi-boot
#+end_src
* NFS+Ganesha
** Container image resource
#+name: Container image
#+begin_src dockerfile :tangle nfs-ganesha/Dockerfile
FROM ubuntu:xenial
LABEL maintainer="Caleb Woodbine <caleb@ii.coop>"
# install prerequisites
RUN DEBIAN_FRONTEND=noninteractive \
&& apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FE869A9 \
&& echo "deb http://ppa.launchpad.net/gluster/nfs-ganesha-2.5/ubuntu xenial main" > /etc/apt/sources.list.d/nfs-ganesha-2.5.list \
&& echo "deb http://ppa.launchpad.net/gluster/libntirpc-1.5/ubuntu xenial main" > /etc/apt/sources.list.d/libntirpc-1.5.list \
&& echo "deb http://ppa.launchpad.net/gluster/glusterfs-3.13/ubuntu xenial main" > /etc/apt/sources.list.d/glusterfs-3.13.list \
&& apt-get update \
&& apt-get install -y netbase nfs-common dbus nfs-ganesha nfs-ganesha-vfs glusterfs-common \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
&& mkdir -p /run/rpcbind /export /var/run/dbus \
&& touch /run/rpcbind/rpcbind.xdr /run/rpcbind/portmap.xdr \
&& chmod 755 /run/rpcbind/* \
&& chown messagebus:messagebus /var/run/dbus
WORKDIR /app
# Add startup script
COPY start.sh /app
RUN chmod +x /app/start.sh
ENV GANESHA_BOOTSTRAP_CONFIG=no
# NFS ports and portmapper
EXPOSE 2049 38465-38467 662 111/udp 111
# Start Ganesha NFS daemon by default
CMD ["/app/start.sh"]
#+end_src
#+name: Start script
#+begin_src sh :tangle nfs-ganesha/start.sh
#!/bin/bash
set -ex
# Options for starting Ganesha
: ${GANESHA_LOGFILE:="/dev/stdout"}
: ${GANESHA_CONFIGFILE:="/etc/ganesha/ganesha.conf"}
: ${GANESHA_OPTIONS:="-N NIV_EVENT"} # NIV_DEBUG
: ${GANESHA_EPOCH:=""}
: ${GANESHA_EXPORT_ID:="77"}
: ${GANESHA_EXPORT:="/var/run/nfs"}
: ${GANESHA_PSEUDO_PATH:="/"}
: ${GANESHA_ACCESS:="*"}
: ${GANESHA_ROOT_ACCESS:="*"}
: ${GANESHA_NFS_PROTOCOLS:="3,4"}
: ${GANESHA_TRANSPORTS:="UDP,TCP"}
: ${GANESHA_BOOTSTRAP_CONFIG:="yes"}
function bootstrap_config {
echo "Bootstrapping Ganesha NFS config"
cat <<END >${GANESHA_CONFIGFILE}
EXPORT
{
# Export Id (mandatory, each EXPORT must have a unique Export_Id)
Export_Id = ${GANESHA_EXPORT_ID};
# Exported path (mandatory)
Path = ${GANESHA_EXPORT};
# Pseudo Path (for NFS v4)
Pseudo = ${GANESHA_PSEUDO_PATH};
# Access control options
Access_Type = RW;
Squash = No_Root_Squash;
Root_Access = "${GANESHA_ROOT_ACCESS}";
Access = "${GANESHA_ACCESS}";
# NFS protocol options
Transports = "${GANESHA_TRANSPORTS}";
Protocols = "${GANESHA_NFS_PROTOCOLS}";
SecType = "sys";
# Exporting FSAL
FSAL {
Name = VFS;
}
}
END
}
function bootstrap_export {
if [ ! -f ${GANESHA_EXPORT} ]; then
mkdir -p "${GANESHA_EXPORT}"
fi
}
function init_rpc {
echo "Starting rpcbind"
rpcbind || return 0
rpc.statd -L || return 0
rpc.idmapd || return 0
sleep 1
}
function init_dbus {
echo "Starting dbus"
rm -f /var/run/dbus/system_bus_socket
rm -f /var/run/dbus/pid
mkdir -p /var/run/dbus
dbus-uuidgen --ensure
dbus-daemon --system --fork
sleep 1
}
function startup_script {
if [ -f "${STARTUP_SCRIPT}" ]; then
/bin/sh ${STARTUP_SCRIPT}
fi
}
if [[ "${GANESHA_BOOTSTRAP_CONFIG}" = "yes" ]]
then
bootstrap_config
fi
bootstrap_export
startup_script
init_rpc
init_dbus
echo "Starting Ganesha NFS"
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib
exec /usr/bin/ganesha.nfsd -F -L ${GANESHA_LOGFILE} -f ${GANESHA_CONFIGFILE} ${GANESHA_OPTIONS}
#+end_src
#+build image
#+begin_src sh
docker build -t ii-pi-boot-nfs-ganesha:latest nfs-ganesha
#+end_src
** Manifests
#+name: NFS+Ganesha DaemonSet
#+begin_src yaml :tangle nfs-ganesha/manifests/daemonset.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: ii-pi-nfs-ganesha
namespace: ii-pi-boot
spec:
selector:
matchLabels:
app: ii-pi-nfs-ganesha
template:
metadata:
labels:
app: ii-pi-nfs-ganesha
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: beta.kubernetes.io/os
operator: In
values:
- linux
- key: beta.kubernetes.io/arch
operator: In
values:
- amd64
- key: ii-pi-boot
operator: In
values:
- "true"
imagePullSecrets:
- name: gitlab-registry
automountServiceAccountToken: false
tolerations:
- operator: Exists
effect: NoSchedule
hostNetwork: true
containers:
- name: ii-pi-nfs-ganesha
image: ii-pi-boot-nfs-ganesha:latest
imagePullPolicy: IfNotPresent
securityContext:
readOnlyRootFilesystem: true
env:
- name: TZ
value: "Pacific/Auckland"
volumeMounts:
- name: nfs-ganesha-config
mountPath: /etc/ganesha
- mountPath: /var/lib/nfs
name: nfs-share
- mountPath: /run
name: run
- mountPath: /var/run
name: var-run
volumes:
- name: nfs-ganesha-config
configMap:
name: nfs-ganesha-config
- name: nfs-share
hostPath:
path: /var/lib/nfs
- name: var-run
emptyDir: {}
- name: run
emptyDir: {}
#+end_src
#+name: Ganesha ConfigMap
#+begin_src yaml :tangle nfs-ganesha/manifests/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: ii-pi-boot
resources:
- daemonset.yaml
configMapGenerator:
- name: nfs-ganesha-config
files:
- ganesha.conf
generatorOptions:
disableNameSuffixHash: true
#+end_src
#+name: Ganesha config
#+begin_src text :tangle nfs-ganesha/manifests/ganesha.conf
EXPORT
{
Export_Id = 106;
Path = /var/lib/nfs/client6;
Pseudo = /var/lib/nfs/client6;
Access_Type = RW;
Protocols = "3";
FSAL {
Name = VFS;
}
CLIENT {
Clients = 192.168.1.106;
Access_Type = "RW";
Squash = No_Root_Squash;
}
}
LOG
{
COMPONENTS {
EXPORT = INFO;
}
}
#+end_src
#+name: Apply manifests
#+begin_src
kubectl apply -k nfs-ganesha/manifests
#+end_src
* TFTP
** Container image resource
#+name: Container image
#+begin_src dockerfile :tangle dnsmasq/Dockerfile
FROM alpine:3.11
LABEL maintainer="Caleb Woodbine <caleb@ii.coop>"
ENV WEBPROC_VERSION 0.2.2
ENV WEBPROC_URL https://github.com/jpillora/webproc/releases/download/$WEBPROC_VERSION/webproc_linux_amd64.gz
# fetch dnsmasq and webproc binary
RUN apk update \
&& apk --no-cache add dnsmasq tftp-hpa \
&& apk add --no-cache --virtual .build-deps curl \
&& curl -sL $WEBPROC_URL | gzip -d - > /usr/local/bin/webproc \
&& chmod +x /usr/local/bin/webproc \
&& apk del .build-deps
#configure dnsmasq
RUN mkdir -p /etc/default/
RUN echo -e "ENABLED=1\nIGNORE_RESOLVCONF=yes" > /etc/default/dnsmasq
COPY dnsmasq.conf /etc/dnsmasq.conf
#run!
ENTRYPOINT ["webproc","--config","/etc/dnsmasq.conf","--","dnsmasq", "--no-daemon"]
#+end_src
#+mame dnsmasq.conf
#+begin_src text :tangle dnsmasq/dnsmasq.conf
port=0
dhcp-range=0.0.0.0,proxy
pxe-service=0,"Raspberry Pi Boot"
enable-tftp
tftp-root=/var/lib/tftpboot
#+end_src
#+name: build image
#+begin_src
docker build -t ii-pi-boot-dnsmasq:latest dnsmasq
#+end_src
** Manifests
#+name: TFTP DaemonSet
#+begin_src yaml :tangle dnsmasq/manifests/daemonset.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: ii-pi-dnsmasq
namespace: ii-pi-boot
spec:
selector:
matchLabels:
app: ii-pi-dnsmasq
template:
metadata:
labels:
app: ii-pi-dnsmasq
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: beta.kubernetes.io/os
operator: In
values:
- linux
- key: beta.kubernetes.io/arch
operator: In
values:
- amd64
- key: ii-pi-boot
operator: In
values:
- "true"
imagePullSecrets:
- name: gitlab-registry
automountServiceAccountToken: false
tolerations:
- operator: Exists
effect: NoSchedule
hostNetwork: true
containers:
- name: ii-pi-dnsmasq
image: ii-pi-boot-dnsmasq:latest
imagePullPolicy: IfNotPresent
securityContext:
readOnlyRootFilesystem: true
env:
- name: TZ
value: "Pacific/Auckland"
volumeMounts:
- mountPath: '/var/lib/tftpboot'
name: tftp-share
- mountPath: '/var/lib/misc'
name: var-lib-misc
initContainers:
- name: tftp-permissions-fix
image: busybox
command: ["/bin/chmod", "-R","0777", "/var/lib/tftpboot"]
volumeMounts:
- name: tftp-share
mountPath: /var/lib/tftpboot
volumes:
- name: tftp-share
hostPath:
path: /var/lib/tftpboot
- name: var-lib-misc
emptyDir: {}
#+end_src
#+name: Apply manifests
#+begin_src
kubectl apply -f dnsmasq/manifests
#+end_src
FROM alpine:3.11
LABEL maintainer="Caleb Woodbine <caleb@ii.coop>"
ENV WEBPROC_VERSION 0.2.2
ENV WEBPROC_URL https://github.com/jpillora/webproc/releases/download/$WEBPROC_VERSION/webproc_linux_amd64.gz
# fetch dnsmasq and webproc binary
RUN apk update \
&& apk --no-cache add dnsmasq tftp-hpa \
&& apk add --no-cache --virtual .build-deps curl \
&& curl -sL $WEBPROC_URL | gzip -d - > /usr/local/bin/webproc \
&& chmod +x /usr/local/bin/webproc \
&& apk del .build-deps
#configure dnsmasq
RUN mkdir -p /etc/default/
RUN echo -e "ENABLED=1\nIGNORE_RESOLVCONF=yes" > /etc/default/dnsmasq
COPY dnsmasq.conf /etc/dnsmasq.conf
#run!
ENTRYPOINT ["webproc","--config","/etc/dnsmasq.conf","--","dnsmasq", "--no-daemon"]
port=0
dhcp-range=0.0.0.0,proxy
pxe-service=0,"Raspberry Pi Boot"
enable-tftp
tftp-root=/var/lib/tftpboot
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: ii-pi-dnsmasq
namespace: ii-pi-boot
spec:
selector:
matchLabels:
app: ii-pi-dnsmasq
template:
metadata:
labels:
app: ii-pi-dnsmasq
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: beta.kubernetes.io/os
operator: In
values:
- linux
- key: beta.kubernetes.io/arch
operator: In
values:
- amd64
- key: ii-pi-boot
operator: In
values:
- "true"
imagePullSecrets:
- name: gitlab-registry
automountServiceAccountToken: false
tolerations:
- operator: Exists
effect: NoSchedule
hostNetwork: true
containers:
- name: ii-pi-dnsmasq
image: ii-pi-boot-dnsmasq:latest
imagePullPolicy: IfNotPresent
securityContext:
readOnlyRootFilesystem: true
env:
- name: TZ
value: "Pacific/Auckland"
volumeMounts:
- mountPath: '/var/lib/tftpboot'
name: tftp-share
- mountPath: '/var/lib/misc'
name: var-lib-misc
initContainers:
- name: tftp-permissions-fix
image: busybox
command: ["/bin/chmod", "-R","0777", "/var/lib/tftpboot"]
volumeMounts:
- name: tftp-share
mountPath: /var/lib/tftpboot
volumes:
- name: tftp-share
hostPath:
path: /var/lib/tftpboot
- name: var-lib-misc
emptyDir: {}
FROM ubuntu:xenial
LABEL maintainer="Caleb Woodbine <caleb@ii.coop>"
# install prerequisites
RUN DEBIAN_FRONTEND=noninteractive \
&& apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FE869A9 \
&& echo "deb http://ppa.launchpad.net/gluster/nfs-ganesha-2.5/ubuntu xenial main" > /etc/apt/sources.list.d/nfs-ganesha-2.5.list \
&& echo "deb http://ppa.launchpad.net/gluster/libntirpc-1.5/ubuntu xenial main" > /etc/apt/sources.list.d/libntirpc-1.5.list \
&& echo "deb http://ppa.launchpad.net/gluster/glusterfs-3.13/ubuntu xenial main" > /etc/apt/sources.list.d/glusterfs-3.13.list \
&& apt-get update \
&& apt-get install -y netbase nfs-common dbus nfs-ganesha nfs-ganesha-vfs glusterfs-common \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
&& mkdir -p /run/rpcbind /export /var/run/dbus \
&& touch /run/rpcbind/rpcbind.xdr /run/rpcbind/portmap.xdr \
&& chmod 755 /run/rpcbind/* \
&& chown messagebus:messagebus /var/run/dbus
WORKDIR /app
# Add startup script
COPY start.sh /app
RUN chmod +x /app/start.sh
ENV GANESHA_BOOTSTRAP_CONFIG=no
# NFS ports and portmapper
EXPOSE 2049 38465-38467 662 111/udp 111
# Start Ganesha NFS daemon by default
CMD ["/app/start.sh"]
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: ii-pi-nfs-ganesha
namespace: ii-pi-boot
spec:
selector:
matchLabels:
app: ii-pi-nfs-ganesha
template:
metadata:
labels:
app: ii-pi-nfs-ganesha
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: beta.kubernetes.io/os
operator: In
values:
- linux
- key: beta.kubernetes.io/arch
operator: In
values:
- amd64
- key: ii-pi-boot
operator: In
values:
- "true"
imagePullSecrets:
- name: gitlab-registry
automountServiceAccountToken: false
tolerations:
- operator: Exists
effect: NoSchedule
hostNetwork: true
containers:
- name: ii-pi-nfs-ganesha
image: ii-pi-boot-nfs-ganesha:latest
imagePullPolicy: IfNotPresent
securityContext:
readOnlyRootFilesystem: true
env:
- name: TZ
value: "Pacific/Auckland"
volumeMounts:
- name: nfs-ganesha-config
mountPath: /etc/ganesha
- mountPath: /var/lib/nfs
name: nfs-share
- mountPath: /run
name: run
- mountPath: /var/run
name: var-run
volumes:
- name: nfs-ganesha-config
configMap:
name: nfs-ganesha-config
- name: nfs-share
hostPath:
path: /var/lib/nfs
- name: var-run
emptyDir: {}
- name: run
emptyDir: {}
EXPORT
{
Export_Id = 106;
Path = /var/lib/nfs/client6;
Pseudo = /var/lib/nfs/client6;
Access_Type = RW;
Protocols = "3";
FSAL {
Name = VFS;
}
CLIENT {
Clients = 192.168.1.106;
Access_Type = "RW";
Squash = No_Root_Squash;
}
}
LOG
{
COMPONENTS {
EXPORT = INFO;
}
}
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: ii-pi-boot
resources:
- daemonset.yaml
configMapGenerator:
- name: nfs-ganesha-config
files:
- ganesha.conf
generatorOptions:
disableNameSuffixHash: true
#!/bin/bash
set -ex
# Options for starting Ganesha
: ${GANESHA_LOGFILE:="/dev/stdout"}