From 3fecb9ca325c070a32370bbdce2c99da5e673571 Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Wed, 15 Jul 2026 10:09:04 +0200 Subject: [PATCH 1/2] USHIFT-7267: Fix OVN stale socket race after upstream flag removal The upstream OVN-K binary removed the --nb-address/--sb-address flags, changing the connection model from TCP to unix sockets. With TCP, each connection attempt was fresh and a missing SBDB caused a clean retry. With unix sockets on a shared hostPath, the socket file persists between container restarts. If ovn-controller connects to a leftover socket from a previous SBDB instance, it caches a stale raft commit index. When the new SBDB starts fresh, ovn-controller sees the lower index as "stale data" and enters an infinite reconnect loop, blocking all pod networking. Fix this with a two-part approach: - SBDB/NBDB containers remove stale sockets before starting ovsdb-server, which then creates fresh ones. - ovn-controller waits for the fresh SBDB socket to appear before connecting, ensuring it never grabs a leftover socket. Applied to both single-node and multi-node DaemonSets. --- assets/components/ovn/multi-node/master/daemonset.yaml | 4 ++++ assets/components/ovn/multi-node/node/daemonset.yaml | 8 ++++++++ assets/components/ovn/single-node/master/daemonset.yaml | 4 ++++ assets/components/ovn/single-node/node/daemonset.yaml | 8 ++++++++ 4 files changed, 24 insertions(+) diff --git a/assets/components/ovn/multi-node/master/daemonset.yaml b/assets/components/ovn/multi-node/master/daemonset.yaml index 38429b1890..1d8ad15d0b 100644 --- a/assets/components/ovn/multi-node/master/daemonset.yaml +++ b/assets/components/ovn/multi-node/master/daemonset.yaml @@ -141,6 +141,8 @@ spec: --db-nb-cluster-local-proto=tcp \ --no-monitor" + rm -f /run/ovn/ovnnb_db.sock + echo "$(date -Iseconds) - starting nbdb" exec /usr/share/ovn/scripts/ovn-ctl \ @@ -286,6 +288,8 @@ spec: --db-sb-cluster-local-proto=tcp \ --no-monitor" + rm -f /run/ovn/ovnsb_db.sock + echo "$(date -Iseconds) - starting sbdb " exec /usr/share/ovn/scripts/ovn-ctl \ ${OVN_ARGS} \ diff --git a/assets/components/ovn/multi-node/node/daemonset.yaml b/assets/components/ovn/multi-node/node/daemonset.yaml index 4d460606b4..7d01ee561d 100644 --- a/assets/components/ovn/multi-node/node/daemonset.yaml +++ b/assets/components/ovn/multi-node/node/daemonset.yaml @@ -55,6 +55,14 @@ spec: # K8S_NODE_IP triggers reconcilation of this daemon when node IP changes echo "$(date -Iseconds) - starting ovn-controller, Node: ${K8S_NODE} IP: ${K8S_NODE_IP}" + # Wait for the SBDB unix socket to appear. The sbdb container + # removes stale sockets and creates fresh ones on startup. + # Connecting to a stale socket would cause ovn-controller to + # cache a raft commit index higher than the fresh SBDB's. + echo "Waiting for SBDB socket..." + while [ ! -S /run/ovn/ovnsb_db.sock ]; do sleep 1; done + echo "SBDB socket ready" + exec ovn-controller unix:/var/run/openvswitch/db.sock -vfile:off \ --no-chdir --pidfile=/var/run/ovn/ovn-controller.pid \ --syslog-method="null" \ diff --git a/assets/components/ovn/single-node/master/daemonset.yaml b/assets/components/ovn/single-node/master/daemonset.yaml index ebabb14610..03f242bd86 100644 --- a/assets/components/ovn/single-node/master/daemonset.yaml +++ b/assets/components/ovn/single-node/master/daemonset.yaml @@ -133,6 +133,8 @@ spec: ovn_db_file="/etc/ovn/ovn${db}_db.db" OVN_ARGS="--db-nb-cluster-local-port=9643 --no-monitor" + rm -f /run/ovn/ovnnb_db.sock + echo "$(date -Iseconds) - starting nbdb" exec /usr/share/ovn/scripts/ovn-ctl \ ${OVN_ARGS} \ @@ -251,6 +253,8 @@ spec: OVN_ARGS="--db-sb-cluster-local-port=9644 --no-monitor" + rm -f /run/ovn/ovnsb_db.sock + echo "$(date -Iseconds) - starting sbdb " exec /usr/share/ovn/scripts/ovn-ctl \ ${OVN_ARGS} \ diff --git a/assets/components/ovn/single-node/node/daemonset.yaml b/assets/components/ovn/single-node/node/daemonset.yaml index 7e7cf95137..1db2968896 100644 --- a/assets/components/ovn/single-node/node/daemonset.yaml +++ b/assets/components/ovn/single-node/node/daemonset.yaml @@ -55,6 +55,14 @@ spec: # K8S_NODE_IP triggers reconcilation of this daemon when node IP changes echo "$(date -Iseconds) - starting ovn-controller, Node: ${K8S_NODE} IP: ${K8S_NODE_IP}" + # Wait for the SBDB unix socket to appear. The sbdb container + # removes stale sockets and creates fresh ones on startup. + # Connecting to a stale socket would cause ovn-controller to + # cache a raft commit index higher than the fresh SBDB's. + echo "Waiting for SBDB socket..." + while [ ! -S /run/ovn/ovnsb_db.sock ]; do sleep 1; done + echo "SBDB socket ready" + exec ovn-controller unix:/var/run/openvswitch/db.sock -vfile:off \ --no-chdir --pidfile=/var/run/ovn/ovn-controller.pid \ --syslog-method="null" \ From 5da8d99355c72d3e98ba72aad89317de8e9afec4 Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Wed, 15 Jul 2026 10:57:56 +0200 Subject: [PATCH 2/2] USHIFT-7267: Use runtime node IP for northd DB addresses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The northd container used template-rendered OVN_NB_DB_LIST and OVN_SB_DB_LIST with the node IP baked into the DaemonSet spec. In multinode, each node rendered the DaemonSet with its own IP, causing spec changes on every reconcile and triggering rolling updates that restarted the SBDB — the root cause of the stale data issue. Replace the baked-in template variables with the runtime env var K8S_NODE_IP, which is resolved per-pod from status.hostIP. This makes the DaemonSet spec identical regardless of which node renders it, eliminating spurious rolling updates. Remove the now-unused Controlplane field from MultiNodeConfig and the OVN_NB_DB_LIST/OVN_SB_DB_LIST render parameters. --- assets/components/ovn/multi-node/master/daemonset.yaml | 6 +++--- pkg/components/networking.go | 4 +--- pkg/config/multinode.go | 4 ---- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/assets/components/ovn/multi-node/master/daemonset.yaml b/assets/components/ovn/multi-node/master/daemonset.yaml index 1d8ad15d0b..fe79c19640 100644 --- a/assets/components/ovn/multi-node/master/daemonset.yaml +++ b/assets/components/ovn/multi-node/master/daemonset.yaml @@ -68,8 +68,8 @@ spec: echo "$(date -Iseconds) - starting ovn-northd" exec ovn-northd \ --no-chdir "-vconsole:${OVN_LOG_LEVEL}" -vfile:off "-vPATTERN:console:%D{%Y-%m-%dT%H:%M:%S.###Z}|%05N|%c%T|%p|%m" \ - --ovnnb-db "{{.OVN_NB_DB_LIST}}" \ - --ovnsb-db "{{.OVN_SB_DB_LIST}}" \ + --ovnnb-db "tcp:${K8S_NODE_IP}:{{.OVN_NB_PORT}}" \ + --ovnsb-db "tcp:${K8S_NODE_IP}:{{.OVN_SB_PORT}}" \ --pidfile /var/run/ovn/ovn-northd.pid & wait $! @@ -179,7 +179,7 @@ spec: done #configure northd_probe_interval - OVN_NB_CTL="ovn-nbctl --db "{{.OVN_NB_DB_LIST}}"" + OVN_NB_CTL="ovn-nbctl --no-leader-only" northd_probe_interval=${OVN_NORTHD_PROBE_INTERVAL:-10000} echo "Setting northd probe interval to ${northd_probe_interval} ms" retries=0 diff --git a/pkg/components/networking.go b/pkg/components/networking.go index 1e54666134..2e6ccad927 100644 --- a/pkg/components/networking.go +++ b/pkg/components/networking.go @@ -110,13 +110,11 @@ func startCNIPlugin(ctx context.Context, cfg *config.Config, kubeconfigPath stri return err } - // Multinode only params: OVN_NB_DB_LIST, OVN_SB_DB_LIST, OVN_NB_PORT, OVN_SB_PORT + // Multinode only params: OVN_NB_PORT, OVN_SB_PORT extraParams := assets.RenderParams{ "OVNConfig": ovnConfig, "KubeconfigPath": kubeconfigPath, "KubeconfigDir": filepath.Join(config.DataDir, "/resources/kubeadmin"), - "OVN_NB_DB_LIST": fmt.Sprintf("tcp:%s:%s", cfg.MultiNode.Controlplane, ovn.OVN_NB_PORT), - "OVN_SB_DB_LIST": fmt.Sprintf("tcp:%s:%s", cfg.MultiNode.Controlplane, ovn.OVN_SB_PORT), "OVN_NB_PORT": ovn.OVN_NB_PORT, "OVN_SB_PORT": ovn.OVN_SB_PORT, } diff --git a/pkg/config/multinode.go b/pkg/config/multinode.go index 6db5e73322..37bd5b773f 100644 --- a/pkg/config/multinode.go +++ b/pkg/config/multinode.go @@ -2,9 +2,6 @@ package config type MultiNodeConfig struct { Enabled bool `json:"enabled"` - // only one controlplane node is supported - // IP address of control plane node - Controlplane string `json:"controlplane"` } // ConfigMultiNode populates multinode configurations to Config.MultiNode @@ -13,6 +10,5 @@ func ConfigMultiNode(c *Config, enabled bool) *Config { return c } c.MultiNode.Enabled = enabled - c.MultiNode.Controlplane = c.Node.NodeIP return c }