Skip to content

Commit a9fedec

Browse files
kvapskvaps
committed
mesh: set src for encapsulated tunl routes
Signed-off-by: Andrei Kvapil <kvapss@gmail.com>
1 parent db4a4a5 commit a9fedec

3 files changed

Lines changed: 148 additions & 1 deletion

File tree

‎manifests/kilo-azure-route-sync-deployment.yaml‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: kilo-azure-route-sync
5+
namespace: cozy-cluster-autoscaler-azure
6+
---
7+
apiVersion: rbac.authorization.k8s.io/v1
8+
kind: ClusterRole
9+
metadata:
10+
name: kilo-azure-route-sync
11+
rules:
12+
- apiGroups: [""]
13+
resources: ["nodes"]
14+
verbs: ["get", "list", "watch"]
15+
---
16+
apiVersion: rbac.authorization.k8s.io/v1
17+
kind: ClusterRoleBinding
18+
metadata:
19+
name: kilo-azure-route-sync
20+
roleRef:
21+
apiGroup: rbac.authorization.k8s.io
22+
kind: ClusterRole
23+
name: kilo-azure-route-sync
24+
subjects:
25+
- kind: ServiceAccount
26+
name: kilo-azure-route-sync
27+
namespace: cozy-cluster-autoscaler-azure
28+
---
29+
apiVersion: apps/v1
30+
kind: Deployment
31+
metadata:
32+
name: kilo-azure-route-sync
33+
namespace: cozy-cluster-autoscaler-azure
34+
spec:
35+
replicas: 1
36+
selector:
37+
matchLabels:
38+
app: kilo-azure-route-sync
39+
template:
40+
metadata:
41+
labels:
42+
app: kilo-azure-route-sync
43+
spec:
44+
serviceAccountName: kilo-azure-route-sync
45+
containers:
46+
- name: sync
47+
image: mcr.microsoft.com/azure-cli:2.67.0
48+
imagePullPolicy: IfNotPresent
49+
env:
50+
- name: AZURE_CLIENT_ID
51+
valueFrom:
52+
secretKeyRef:
53+
name: cluster-autoscaler-azure-azure-cluster-autoscaler
54+
key: ClientID
55+
- name: AZURE_CLIENT_SECRET
56+
valueFrom:
57+
secretKeyRef:
58+
name: cluster-autoscaler-azure-azure-cluster-autoscaler
59+
key: ClientSecret
60+
- name: AZURE_TENANT_ID
61+
valueFrom:
62+
secretKeyRef:
63+
name: cluster-autoscaler-azure-azure-cluster-autoscaler
64+
key: TenantID
65+
- name: AZURE_SUBSCRIPTION_ID
66+
valueFrom:
67+
secretKeyRef:
68+
name: cluster-autoscaler-azure-azure-cluster-autoscaler
69+
key: SubscriptionID
70+
- name: AZURE_RESOURCE_GROUP
71+
valueFrom:
72+
secretKeyRef:
73+
name: cluster-autoscaler-azure-azure-cluster-autoscaler
74+
key: ResourceGroup
75+
- name: AZURE_ROUTE_TABLE
76+
value: kilo-routes-workers-serverscom
77+
- name: AZURE_VNET_NAME
78+
value: cozystack-vnet
79+
- name: AZURE_SUBNET_NAME
80+
value: workers-serverscom
81+
- name: AZURE_ROUTES
82+
value: to-serverscom=192.168.102.0/23
83+
command: ["/bin/sh","-ceu"]
84+
args:
85+
- |
86+
az login --service-principal -u "$AZURE_CLIENT_ID" -p "$AZURE_CLIENT_SECRET" --tenant "$AZURE_TENANT_ID" >/dev/null
87+
az account set --subscription "$AZURE_SUBSCRIPTION_ID"
88+
89+
az aks install-cli --install-location /usr/local/bin/kubectl >/dev/null
90+
91+
sync_route() {
92+
route_name="$1"
93+
route_prefix="$2"
94+
leader_ip="$3"
95+
az network route-table route create -g "$AZURE_RESOURCE_GROUP" --route-table-name "$AZURE_ROUTE_TABLE" \
96+
-n "$route_name" --address-prefix "$route_prefix" \
97+
--next-hop-type VirtualAppliance --next-hop-ip-address "$leader_ip" >/dev/null || true
98+
az network route-table route update -g "$AZURE_RESOURCE_GROUP" --route-table-name "$AZURE_ROUTE_TABLE" \
99+
-n "$route_name" --address-prefix "$route_prefix" \
100+
--next-hop-type VirtualAppliance --next-hop-ip-address "$leader_ip" >/dev/null
101+
}
102+
103+
sync_all_routes() {
104+
leader_ip="$1"
105+
IFS=','
106+
for entry in $AZURE_ROUTES; do
107+
route_name="${entry%%=*}"
108+
route_prefix="${entry#*=}"
109+
[ -n "$route_name" ] && [ -n "$route_prefix" ] || continue
110+
sync_route "$route_name" "$route_prefix" "$leader_ip"
111+
done
112+
unset IFS
113+
}
114+
115+
kubectl get node -w -l topology.kubernetes.io/zone=azure --no-headers \
116+
-o 'custom-columns=NAME:.metadata.name,LEADER:.metadata.annotations.kilo\.squat\.ai/leader,IP:.status.addresses[?(@.type=="InternalIP")].address' \
117+
| while read -r n leader ip; do
118+
echo "$(date -Iseconds) event node=${n} leader=${leader} ip=${ip}"
119+
[ "$leader" = "true" ] || continue
120+
az network vnet subnet update \
121+
-g "$AZURE_RESOURCE_GROUP" \
122+
--vnet-name "$AZURE_VNET_NAME" \
123+
-n "$AZURE_SUBNET_NAME" \
124+
--route-table "$AZURE_ROUTE_TABLE" >/dev/null
125+
126+
sync_all_routes "$ip"
127+
128+
echo "$(date -Iseconds) synced routes to leader ${n} (${ip})"
129+
done

‎pkg/mesh/routes.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
7676
Flags: int(netlink.FLAG_ONLINK),
7777
Gw: segment.privateIPs[i],
7878
LinkIndex: tunlIface,
79+
Src: t.privateIP.IP,
7980
Protocol: unix.RTPROT_STATIC,
8081
Table: kiloTableIndex,
8182
})
@@ -169,6 +170,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
169170
Flags: int(netlink.FLAG_ONLINK),
170171
Gw: segment.privateIPs[i],
171172
LinkIndex: tunlIface,
173+
Src: t.privateIP.IP,
172174
Protocol: unix.RTPROT_STATIC,
173175
Table: kiloTableIndex,
174176
})
@@ -316,8 +318,11 @@ func (t *Topology) PeerRoutes(name string, kiloIface int, additionalAllowedIPs [
316318
}
317319

318320
func encapsulateRoute(route *netlink.Route, encapsulate encapsulation.Strategy, subnet *net.IPNet, tunlIface int) *netlink.Route {
319-
if encapsulate == encapsulation.Always || (encapsulate == encapsulation.CrossSubnet && !subnet.Contains(route.Gw)) {
321+
if encapsulate == encapsulation.Always || (encapsulate == encapsulation.CrossSubnet && subnet != nil && !subnet.Contains(route.Gw)) {
320322
route.LinkIndex = tunlIface
323+
if subnet != nil && route.Src == nil {
324+
route.Src = subnet.IP
325+
}
321326
}
322327
return route
323328
}

‎pkg/mesh/routes_test.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -747,13 +747,15 @@ func TestRoutes(t *testing.T) {
747747
Flags: int(netlink.FLAG_ONLINK),
748748
Gw: nodes["c"].InternalIP.IP,
749749
LinkIndex: tunlIface,
750+
Src: nodes["b"].InternalIP.IP,
750751
Protocol: unix.RTPROT_STATIC,
751752
},
752753
{
753754
Dst: oneAddressCIDR(nodes["c"].InternalIP.IP),
754755
Flags: int(netlink.FLAG_ONLINK),
755756
Gw: nodes["c"].InternalIP.IP,
756757
LinkIndex: tunlIface,
758+
Src: nodes["b"].InternalIP.IP,
757759
Protocol: unix.RTPROT_STATIC,
758760
Table: kiloTableIndex,
759761
},
@@ -886,41 +888,47 @@ func TestRoutes(t *testing.T) {
886888
Flags: int(netlink.FLAG_ONLINK),
887889
Gw: nodes["b"].InternalIP.IP,
888890
LinkIndex: tunlIface,
891+
Src: nodes["c"].InternalIP.IP,
889892
Protocol: unix.RTPROT_STATIC,
890893
},
891894
{
892895
Dst: nodes["a"].Subnet,
893896
Flags: int(netlink.FLAG_ONLINK),
894897
Gw: nodes["b"].InternalIP.IP,
895898
LinkIndex: tunlIface,
899+
Src: nodes["c"].InternalIP.IP,
896900
Protocol: unix.RTPROT_STATIC,
897901
},
898902
{
899903
Dst: oneAddressCIDR(nodes["a"].InternalIP.IP),
900904
Flags: int(netlink.FLAG_ONLINK),
901905
Gw: nodes["b"].InternalIP.IP,
902906
LinkIndex: tunlIface,
907+
Src: nodes["c"].InternalIP.IP,
903908
Protocol: unix.RTPROT_STATIC,
904909
},
905910
{
906911
Dst: oneAddressCIDR(mustTopoForGranularityAndHost(LogicalGranularity, nodes["c"].Name).segments[1].wireGuardIP),
907912
Flags: int(netlink.FLAG_ONLINK),
908913
Gw: nodes["b"].InternalIP.IP,
909914
LinkIndex: tunlIface,
915+
Src: nodes["c"].InternalIP.IP,
910916
Protocol: unix.RTPROT_STATIC,
911917
},
912918
{
913919
Dst: nodes["b"].Subnet,
914920
Flags: int(netlink.FLAG_ONLINK),
915921
Gw: nodes["b"].InternalIP.IP,
916922
LinkIndex: tunlIface,
923+
Src: nodes["c"].InternalIP.IP,
917924
Protocol: unix.RTPROT_STATIC,
918925
},
919926
{
920927
Dst: nodes["b"].InternalIP,
921928
Flags: int(netlink.FLAG_ONLINK),
922929
Gw: nodes["b"].InternalIP.IP,
923930
LinkIndex: tunlIface,
931+
Src: nodes["c"].InternalIP.IP,
924932
Protocol: unix.RTPROT_STATIC,
925933
Table: kiloTableIndex,
926934
},
@@ -929,34 +937,39 @@ func TestRoutes(t *testing.T) {
929937
Flags: int(netlink.FLAG_ONLINK),
930938
Gw: nodes["b"].InternalIP.IP,
931939
LinkIndex: tunlIface,
940+
Src: nodes["c"].InternalIP.IP,
932941
Protocol: unix.RTPROT_STATIC,
933942
},
934943
{
935944
Dst: nodes["d"].Subnet,
936945
Flags: int(netlink.FLAG_ONLINK),
937946
Gw: nodes["b"].InternalIP.IP,
938947
LinkIndex: tunlIface,
948+
Src: nodes["c"].InternalIP.IP,
939949
Protocol: unix.RTPROT_STATIC,
940950
},
941951
{
942952
Dst: &peers["a"].AllowedIPs[0],
943953
Flags: int(netlink.FLAG_ONLINK),
944954
Gw: nodes["b"].InternalIP.IP,
945955
LinkIndex: tunlIface,
956+
Src: nodes["c"].InternalIP.IP,
946957
Protocol: unix.RTPROT_STATIC,
947958
},
948959
{
949960
Dst: &peers["a"].AllowedIPs[1],
950961
Flags: int(netlink.FLAG_ONLINK),
951962
Gw: nodes["b"].InternalIP.IP,
952963
LinkIndex: tunlIface,
964+
Src: nodes["c"].InternalIP.IP,
953965
Protocol: unix.RTPROT_STATIC,
954966
},
955967
{
956968
Dst: &peers["b"].AllowedIPs[0],
957969
Flags: int(netlink.FLAG_ONLINK),
958970
Gw: nodes["b"].InternalIP.IP,
959971
LinkIndex: tunlIface,
972+
Src: nodes["c"].InternalIP.IP,
960973
Protocol: unix.RTPROT_STATIC,
961974
},
962975
},

0 commit comments

Comments
 (0)