Doing it the hard way
go to https://console.redhat.com/openshift/install/metal/user-provisioned
Download all the things
https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/stable/openshift-install-linux.tar.gz
https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/stable/openshift-client-linux.tar.gz
https://mirror.openshift.com/pub/openshift-v4/x86_64/dependencies/rhcos/latest/rhcos-live.x86_64.iso
https://mirror.openshift.com/pub/openshift-v4/x86_64/dependencies/rhcos/latest/rhcos-live-rootfs.x86_64.img
Setup dns w/ ptr
kinit admin
ipa dnszone-add --name-from-ip=10.9.8.0/24
ipa dnsrecord-add rockyhowto.lab api.rockycluster --ttl=3600 --a-ip-address=10.9.8.1
ipa dnsrecord-add 8.9.10.in-addr.arpa. 1 --ptr-rec api.rockycluster.rockyhowto.lab.
ipa dnsrecord-add rockyhowto.lab api-int.rockycluster --ttl=3600 --a-ip-address=10.9.8.1
ipa dnsrecord-add 8.9.10.in-addr.arpa. 1 --ptr-rec api-int.rockycluster.rockyhowto.lab.
ipa dnsrecord-add rockyhowto.lab *.apps.rockycluster --ttl=3600 --a-ip-address=10.9.8.1
ipa dnsrecord-add rockyhowto.lab node01.rockycluster --ttl=3600 --a-ip-address=10.9.8.110
ipa dnsrecord-add 8.9.10.in-addr.arpa. 110 --ptr-rec node01.rockycluster.rockyhowto.lab.
ipa dnsrecord-add rockyhowto.lab node02.rockycluster --ttl=3600 --a-ip-address=10.9.8.120
ipa dnsrecord-add 8.9.10.in-addr.arpa. 120 --ptr-rec node02.rockycluster.rockyhowto.lab.
ipa dnsrecord-add rockyhowto.lab node03.rockycluster --ttl=3600 --a-ip-address=10.9.8.130
ipa dnsrecord-add 8.9.10.in-addr.arpa. 130 --ptr-rec node03.rockycluster.rockyhowto.lab.
ipa dnsrecord-add rockyhowto.lab node04.rockycluster --ttl=3600 --a-ip-address=10.9.8.140
ipa dnsrecord-add 8.9.10.in-addr.arpa. 140 --ptr-rec node04.rockycluster.rockyhowto.lab.
ipa dnsrecord-add rockyhowto.lab compute0.rockycluster --ttl=3600 --a-ip-address=10.9.8.151
ipa dnsrecord-add 8.9.10.in-addr.arpa. 151 --ptr-rec compute0.rockycluster.rockyhowto.lab.
ipa dnsrecord-add rockyhowto.lab compute1.rockycluster --ttl=3600 --a-ip-address=10.9.8.152
ipa dnsrecord-add 8.9.10.in-addr.arpa. 152 --ptr-rec compute0.rockycluster.rockyhowto.lab.
Check dns entries
dig +noall +answer @svc01 random.apps.rockycluster.rockyhowto.lab
dig +noall +answer @svc01 -x 10.9.8.1
Setup load balancer and firewall
dnf -y install haproxy
setsebool -P haproxy_connect_any 1
firewall-cmd --permanent --add-port=80/tcp --zone=internal
firewall-cmd --permanent --add-port=443/tcp --zone=internal
firewall-cmd --permanent --add-port=6443/tcp --zone=internal
firewall-cmd --permanent --add-port=22623/tcp --zone=internal
firewall-cmd --reload
systemctl restart firewalld
Make /etc/haproxy/haproxy.cfg
tee /etc/haproxy/haproxy.cfg > /dev/null << 'EOT'
global
log 127.0.0.1 local2
pidfile /var/run/haproxy.pid
maxconn 4000
daemon
defaults
mode http
log global
option dontlognull
option http-server-close
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 1m
timeout server 1m
timeout http-keep-alive 10s
timeout check 10s
maxconn 3000
listen api-server-6443
bind *:6443
mode tcp
option httpchk GET /readyz HTTP/1.0
option log-health-checks
balance roundrobin
server bootstrap bootstrap.rockycluster.rockyhowto.lab:6443 verify none check check-ssl inter 10s fall 2 rise 3 backup
server control-plane0 control-plane0.rockycluster.rockyhowto.lab:6443 weight 1 verify none check check-ssl inter 10s fall 2 rise 3
server control-plane1 control-plane1.rockycluster.rockyhowto.lab:6443 weight 1 verify none check check-ssl inter 10s fall 2 rise 3
server control-plane2 control-plane2.rockycluster.rockyhowto.lab:6443 weight 1 verify none check check-ssl inter 10s fall 2 rise 3
listen machine-config-server-22623
bind *:22623
mode tcp
server bootstrap bootstrap.rockycluster.rockyhowto.lab:22623 check inter 1s backup
server control-plane0 control-plane0.rockycluster.rockyhowto.lab:22623 check inter 1s
server control-plane1 control-plane1.rockycluster.rockyhowto.lab:22623 check inter 1s
server control-plane2 control-plane2.rockycluster.rockyhowto.lab:22623 check inter 1s
listen ingress-router-443
bind *:443
mode tcp
balance source
server compute0 compute0.rockycluster.rockyhowto.lab:443 check inter 1s
server compute1 compute1.rockycluster.rockyhowto.lab:443 check inter 1s
listen ingress-router-80
bind *:80
mode tcp
balance source
server compute0 compute0.rockycluster.rockyhowto.lab:80 check inter 1s
server compute1 compute1.rockycluster.rockyhowto.lab:80 check inter 1s
EOT
Restart HAproxy
haproxy -f /etc/haproxy/haproxy.cfg -c
sudo systemctl enable haproxy
sudo systemctl restart haproxy
sudo systemctl status haproxy
journalctl -xeu haproxy.service
On cwo02
make a key to ssh into controle plane nodes
ssh-keygen -t ed25519 -N '' -f key
untar install file
tar -xvf openshift-install-linux.tar.gz
untar cli file
tar -xzf openshift-client-linux.tar.gz
Place the oc binary in a directory that is on your /usr/bin.
cp oc /usr/bin/
Create the working yaml for the install.
tee /home/cluster/install-config.yaml > /dev/null << 'EOT'
apiVersion: v1
baseDomain: rockyhowto.lab
compute:
- hyperthreading: Enabled
name: worker
replicas: 0
controlPlane:
hyperthreading: Enabled
name: control-plane
replicas: 3
metadata:
name: rockycluster
networking:
clusterNetwork:
- cidr: 10.128.0.0/14
hostPrefix: 23
networkType: OVNKubernetes
serviceNetwork:
- 172.30.0.0/16
platform:
none: {}
fips: false
pullSecret: '{"auths":{"cloud.openshift.com":{"auth":"b3BlbnNoaWZ0LXJlbGVhc2UtZGV2K29jbV9hY2Nlc3NfYWZmMjkwZDY2MjMzNGVhNjk5NmQwMTdjMDhjODc3YTM6SzUwSERGOEpBM1pGSkZJREJTWU05Nzk1NkEyNko4TVVFQzU2ODNDQ1VLUUhUQ0tJSkdGTFNMSkVKTDEyMkpZSQ==","email":"john.f.belcher@gmail.com"},"quay.io":{"auth":"b3BlbnNoaWZ0LXJlbGVhc2UtZGV2K29jbV9hY2Nlc3NfYWZmMjkwZDY2MjMzNGVhNjk5NmQwMTdjMDhjODc3YTM6SzUwSERGOEpBM1pGSkZJREJTWU05Nzk1NkEyNko4TVVFQzU2ODNDQ1VLUUhUQ0tJSkdGTFNMSkVKTDEyMkpZSQ==","email":"john.f.belcher@gmail.com"},"registry.connect.redhat.com":{"auth":"fHVoYy1wb29sLTU2NWVkZDg1LWIwYWYtNGNjMy1iMzU2LWYyNDUxODZjYWYyMTpleUpoYkdjaU9pSlNVelV4TWlKOS5leUp6ZFdJaU9pSmpOV1U1TnpVNE0yVTRPVFkwTkdZMllqRmhPVFF4WVdGbE9UUm1OekE0WWlKOS5IWlJlZDJESnJnVi1IV01TUUFPbFFrcHA2YUJsMEdZdHpYVVlzWjl3RUtEYkpTYW54WWR1MXNZcFJRTmRkQVY5MF95dGUtMzR0QmoxSU04dDl3N2U0VmlXN1ROTUVVVnVsVFNmNkdoWU1NWmMwMVN1YUVsRHhxQmRGZFljM2lWWjRYLVBCQ3BCNUE0U21TVURqR1VaZmZPbHplazJpUGtPeUprcU55TkRMdkdpUFkwR2RDczhHd3JVckxNZndHMnNfQWZoVEpWcHNTeEFzUnVpTEcySFVuWi0wLW1kWEFVUU5YYXlyT0F4azRxcXVFMkVkZlh3a25jR2JJN0FtNU5peG5HZnJISDJtVnlRTmRsOVl4TENqMVB6REsyMUlwSkI5VDg5RGxEaWl4S05tb04zUFI1VlYzd3FEY3dnQkZmMWpCZlE2Ul8yZVAxaTN4UjFZN3pnT3hoclJRN2ZaR3g2Y215enBVT2p6bHA5bEVnMlBWZHhDZ3hoTmJPYl9VVC04em5ieWxOMEdza3FieklPeGJ1eExVXzI0eF9qX3YxODhJM2ZVWkg4RHlsT2dqZTlrbEhQTDVEek5vNXh1NTVpS3UyUDh5R2VfdkRtamppZWkzNWNROUcxSDlDRlVvNDN1V3AzQ1IyVUlJbWt4ZXUxRENGSFVYNnp6OGNIZ2hxaDlnam5nSm5wUGRDR2tmZEhYb2JPd1BjaFZTdnI2bVprLS1YX0l2c0pKdW1Td0ZqVWN0eW1felNnUVBXRHFDXzV4d0xLX1NRQUItbEhpVmJVem1UeHRyMV9uVTVNNlB5bm5iLS00Q0hkVDhGLWF4dnhTOWRLVjNEcHNYdkNnSXM1YmV5TkxkdnY0NEdEay1PM040ZkxuakI1Y05na05Ya1o1aTBlMzlvMGVWOA==","email":"john.f.belcher@gmail.com"},"registry.redhat.io":{"auth":"fHVoYy1wb29sLTU2NWVkZDg1LWIwYWYtNGNjMy1iMzU2LWYyNDUxODZjYWYyMTpleUpoYkdjaU9pSlNVelV4TWlKOS5leUp6ZFdJaU9pSmpOV1U1TnpVNE0yVTRPVFkwTkdZMllqRmhPVFF4WVdGbE9UUm1OekE0WWlKOS5IWlJlZDJESnJnVi1IV01TUUFPbFFrcHA2YUJsMEdZdHpYVVlzWjl3RUtEYkpTYW54WWR1MXNZcFJRTmRkQVY5MF95dGUtMzR0QmoxSU04dDl3N2U0VmlXN1ROTUVVVnVsVFNmNkdoWU1NWmMwMVN1YUVsRHhxQmRGZFljM2lWWjRYLVBCQ3BCNUE0U21TVURqR1VaZmZPbHplazJpUGtPeUprcU55TkRMdkdpUFkwR2RDczhHd3JVckxNZndHMnNfQWZoVEpWcHNTeEFzUnVpTEcySFVuWi0wLW1kWEFVUU5YYXlyT0F4azRxcXVFMkVkZlh3a25jR2JJN0FtNU5peG5HZnJISDJtVnlRTmRsOVl4TENqMVB6REsyMUlwSkI5VDg5RGxEaWl4S05tb04zUFI1VlYzd3FEY3dnQkZmMWpCZlE2Ul8yZVAxaTN4UjFZN3pnT3hoclJRN2ZaR3g2Y215enBVT2p6bHA5bEVnMlBWZHhDZ3hoTmJPYl9VVC04em5ieWxOMEdza3FieklPeGJ1eExVXzI0eF9qX3YxODhJM2ZVWkg4RHlsT2dqZTlrbEhQTDVEek5vNXh1NTVpS3UyUDh5R2VfdkRtamppZWkzNWNROUcxSDlDRlVvNDN1V3AzQ1IyVUlJbWt4ZXUxRENGSFVYNnp6OGNIZ2hxaDlnam5nSm5wUGRDR2tmZEhYb2JPd1BjaFZTdnI2bVprLS1YX0l2c0pKdW1Td0ZqVWN0eW1felNnUVBXRHFDXzV4d0xLX1NRQUItbEhpVmJVem1UeHRyMV9uVTVNNlB5bm5iLS00Q0hkVDhGLWF4dnhTOWRLVjNEcHNYdkNnSXM1YmV5TkxkdnY0NEdEay1PM040ZkxuakI1Y05na05Ya1o1aTBlMzlvMGVWOA==","email":"john.f.belcher@gmail.com"}}}'
sshKey: 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFdX0p9rQ9gbv20LhMXgsv6+acBoxGN6SOlwHiZ2mKy4'
EOT
Creating the Kubernetes manifest and Ignition config files
mkdir /home/cluster/clusterinstall
\cp -v /home/cluster/install-config.yaml /home/cluster/clusterinstall/
./openshift-install create manifests --dir /home/cluster/clusterinstall
Create the Ignition configuration files
./openshift-install create ignition-configs --dir /home/cluster/clusterinstall
Move ign file to web server location
mkdir /home/tftpboot/openshift
\cp -v /home/cluster/clusterinstall/*.ign /home/tftpboot/openshift
\cp -v /home/cluster/*.img /home/tftpboot/openshift
\cp -v /home/cluster/rhcos-live-kernel-x86_64 /home/tftpboot/openshift
Check if the server is presenting the new files
curl -i http://10.9.8.10/tftpboot/openshift/rhcos-live-kernel-x86_64
Go PXE boot the cluster machines. Be sure that the installation is successful on each node before commencing with the OpenShift Container Platform installation. Observing the installation process can also help to determine the cause of RHCOS installation issues that might arise.
Monitor the bootstrap process.
./openshift-install --dir clusterinstall wait-for bootstrap-complete --log-level=debug
ssh -o "IdentitiesOnly=yes" -i key core@bootstrap.rockycluster.rockyhowto.lab
sudo podman image list --all
sudo podman container list --all
for pod in $(sudo podman ps -a -q); do sudo podman logs $pod; done
Confirm the cp nodes are in the cluster
export KUBECONFIG=/home/cluster/clusterinstall/auth/kubeconfig
oc whoami
oc get nodes
check and approve csr
oc get csr
oc adm certificate approve <csr_name>
Watch the cluster components come online:
watch -n5 oc get clusteroperators