Documentation
¶
Index ¶
Constants ¶
This section is empty.
Variables ¶
View Source
var ( KubeOvnCrd = template.Must(template.New("kube-ovn-crd.yaml").Parse( dedent.Dedent(`--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: vpc-nat-gateways.kubeovn.io spec: group: kubeovn.io names: plural: vpc-nat-gateways singular: vpc-nat-gateway shortNames: - vpc-nat-gw kind: VpcNatGateway listKind: VpcNatGatewayList scope: Cluster versions: - additionalPrinterColumns: - jsonPath: .spec.vpc name: Vpc type: string - jsonPath: .spec.subnet name: Subnet type: string - jsonPath: .spec.lanIp name: LanIP type: string name: v1 served: true storage: true schema: openAPIV3Schema: type: object properties: spec: type: object properties: lanIp: type: string subnet: type: string vpc: type: string selector: type: array items: type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: iptables-eips.kubeovn.io spec: group: kubeovn.io names: plural: iptables-eips singular: iptables-eip shortNames: - eip kind: IptablesEIP listKind: IptablesEIPList scope: Cluster versions: - name: v1 served: true storage: true subresources: status: {} additionalPrinterColumns: - jsonPath: .status.ip name: IP type: string - jsonPath: .spec.macAddress name: Mac type: string - jsonPath: .status.nat name: Nat type: string - jsonPath: .spec.natGwDp name: NatGwDp type: string - jsonPath: .status.ready name: Ready type: boolean schema: openAPIV3Schema: type: object properties: status: type: object properties: ready: type: boolean ip: type: string nat: type: string redo: type: string conditions: type: array items: type: object properties: type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string spec: type: object properties: v4ip: type: string v6ip: type: string macAddress: type: string natGwDp: type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: iptables-fip-rules.kubeovn.io spec: group: kubeovn.io names: plural: iptables-fip-rules singular: iptables-fip-rule shortNames: - fip kind: IptablesFIPRule listKind: IptablesFIPRuleList scope: Cluster versions: - name: v1 served: true storage: true subresources: status: {} additionalPrinterColumns: - jsonPath: .spec.eip name: Eip type: string - jsonPath: .status.v4ip name: V4ip type: string - jsonPath: .spec.internalIp name: InternalIp type: string - jsonPath: .status.v6ip name: V6ip type: string - jsonPath: .status.ready name: Ready type: boolean - jsonPath: .status.natGwDp name: NatGwDp type: string schema: openAPIV3Schema: type: object properties: status: type: object properties: ready: type: boolean v4ip: type: string v6ip: type: string natGwDp: type: string redo: type: string conditions: type: array items: type: object properties: type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string spec: type: object properties: eip: type: string internalIp: type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: iptables-dnat-rules.kubeovn.io spec: group: kubeovn.io names: plural: iptables-dnat-rules singular: iptables-dnat-rule shortNames: - dnat kind: IptablesDnatRule listKind: IptablesDnatRuleList scope: Cluster versions: - name: v1 served: true storage: true subresources: status: {} additionalPrinterColumns: - jsonPath: .spec.eip name: Eip type: string - jsonPath: .spec.protocol name: Protocol type: string - jsonPath: .status.v4ip name: V4ip type: string - jsonPath: .status.v6ip name: V6ip type: string - jsonPath: .spec.internalIp name: InternalIp type: string - jsonPath: .spec.externalPort name: ExternalPort type: string - jsonPath: .spec.internalPort name: InternalPort type: string - jsonPath: .status.natGwDp name: NatGwDp type: string - jsonPath: .status.ready name: Ready type: boolean schema: openAPIV3Schema: type: object properties: status: type: object properties: ready: type: boolean v4ip: type: string v6ip: type: string natGwDp: type: string redo: type: string conditions: type: array items: type: object properties: type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string spec: type: object properties: eip: type: string externalPort: type: string protocol: type: string internalIp: type: string internalPort: type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: iptables-snat-rules.kubeovn.io spec: group: kubeovn.io names: plural: iptables-snat-rules singular: iptables-snat-rule shortNames: - snat kind: IptablesSnatRule listKind: IptablesSnatRuleList scope: Cluster versions: - name: v1 served: true storage: true subresources: status: {} additionalPrinterColumns: - jsonPath: .spec.eip name: EIP type: string - jsonPath: .status.v4ip name: V4ip type: string - jsonPath: .status.v6ip name: V6ip type: string - jsonPath: .spec.internalCIDR name: InternalCIDR type: string - jsonPath: .status.natGwDp name: NatGwDp type: string - jsonPath: .status.ready name: Ready type: boolean schema: openAPIV3Schema: type: object properties: status: type: object properties: ready: type: boolean v4ip: type: string v6ip: type: string natGwDp: type: string redo: type: string conditions: type: array items: type: object properties: type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string spec: type: object properties: eip: type: string internalCIDR: type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: vpcs.kubeovn.io spec: group: kubeovn.io versions: - additionalPrinterColumns: - jsonPath: .status.standby name: Standby type: boolean - jsonPath: .status.subnets name: Subnets type: string - jsonPath: .spec.namespaces name: Namespaces type: string name: v1 schema: openAPIV3Schema: properties: spec: properties: namespaces: items: type: string type: array staticRoutes: items: properties: policy: type: string cidr: type: string nextHopIP: type: string type: object type: array policyRoutes: items: properties: priority: type: integer action: type: string match: type: string nextHopIP: type: string type: object type: array vpcPeerings: items: properties: remoteVpc: type: string localConnectIP: type: string type: object type: array type: object status: properties: conditions: items: properties: lastTransitionTime: type: string lastUpdateTime: type: string message: type: string reason: type: string status: type: string type: type: string type: object type: array default: type: boolean defaultLogicalSwitch: type: string router: type: string standby: type: boolean subnets: items: type: string type: array vpcPeerings: items: type: string type: array tcpLoadBalancer: type: string tcpSessionLoadBalancer: type: string udpLoadBalancer: type: string udpSessionLoadBalancer: type: string type: object type: object served: true storage: true subresources: status: {} names: kind: Vpc listKind: VpcList plural: vpcs shortNames: - vpc singular: vpc scope: Cluster --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: ips.kubeovn.io spec: group: kubeovn.io versions: - name: v1 served: true storage: true additionalPrinterColumns: - name: V4IP type: string jsonPath: .spec.v4IpAddress - name: V6IP type: string jsonPath: .spec.v6IpAddress - name: Mac type: string jsonPath: .spec.macAddress - name: Node type: string jsonPath: .spec.nodeName - name: Subnet type: string jsonPath: .spec.subnet schema: openAPIV3Schema: type: object properties: spec: type: object properties: podName: type: string namespace: type: string subnet: type: string attachSubnets: type: array items: type: string nodeName: type: string ipAddress: type: string v4IpAddress: type: string v6IpAddress: type: string attachIps: type: array items: type: string macAddress: type: string attachMacs: type: array items: type: string containerID: type: string podType: type: string scope: Cluster names: plural: ips singular: ip kind: IP shortNames: - ip --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: vips.kubeovn.io spec: group: kubeovn.io names: plural: vips singular: vip shortNames: - vip kind: Vip listKind: VipList scope: Cluster versions: - name: v1 served: true storage: true additionalPrinterColumns: - name: V4IP type: string jsonPath: .spec.v4ip - name: PV4IP type: string jsonPath: .spec.parentV4ip - name: Mac type: string jsonPath: .spec.macAddress - name: PMac type: string jsonPath: .spec.ParentMac - name: V6IP type: string jsonPath: .spec.v6ip - name: PV6IP type: string jsonPath: .spec.parentV6ip - name: Subnet type: string jsonPath: .spec.subnet - jsonPath: .status.ready name: Ready type: boolean schema: openAPIV3Schema: type: object properties: status: type: object properties: ready: type: boolean v4ip: type: string v6ip: type: string mac: type: string pv4ip: type: string pv6ip: type: string pmac: type: string conditions: type: array items: type: object properties: type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string spec: type: object properties: namespace: type: string subnet: type: string attachSubnets: type: array items: type: string v4ip: type: string macAddress: type: string v6ip: type: string parentV4ip: type: string parentMac: type: string parentV6ip: type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: subnets.kubeovn.io spec: group: kubeovn.io versions: - name: v1 served: true storage: true subresources: status: {} additionalPrinterColumns: - name: Provider type: string jsonPath: .spec.provider - name: Vpc type: string jsonPath: .spec.vpc - name: Protocol type: string jsonPath: .spec.protocol - name: CIDR type: string jsonPath: .spec.cidrBlock - name: Private type: boolean jsonPath: .spec.private - name: NAT type: boolean jsonPath: .spec.natOutgoing - name: Default type: boolean jsonPath: .spec.default - name: GatewayType type: string jsonPath: .spec.gatewayType - name: V4Used type: number jsonPath: .status.v4usingIPs - name: V4Available type: number jsonPath: .status.v4availableIPs - name: V6Used type: number jsonPath: .status.v6usingIPs - name: V6Available type: number jsonPath: .status.v6availableIPs - name: ExcludeIPs type: string jsonPath: .spec.excludeIps schema: openAPIV3Schema: type: object properties: status: type: object properties: v4availableIPs: type: number v4usingIPs: type: number v6availableIPs: type: number v6usingIPs: type: number activateGateway: type: string dhcpV4OptionsUUID: type: string dhcpV6OptionsUUID: type: string conditions: type: array items: type: object properties: type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string spec: type: object properties: vpc: type: string default: type: boolean protocol: type: string enum: - IPv4 - IPv6 - Dual cidrBlock: type: string namespaces: type: array items: type: string gateway: type: string provider: type: string excludeIps: type: array items: type: string vips: type: array items: type: string gatewayType: type: string allowSubnets: type: array items: type: string gatewayNode: type: string natOutgoing: type: boolean externalEgressGateway: type: string policyRoutingPriority: type: integer minimum: 1 maximum: 32765 policyRoutingTableID: type: integer minimum: 1 maximum: 2147483647 not: enum: - 252 # compat - 253 # default - 254 # main - 255 # local private: type: boolean vlan: type: string logicalGateway: type: boolean disableGatewayCheck: type: boolean disableInterConnection: type: boolean htbqos: type: string enableDHCP: type: boolean dhcpV4Options: type: string dhcpV6Options: type: string enableIPv6RA: type: boolean ipv6RAConfigs: type: string acls: type: array items: type: object properties: direction: type: string enum: - from-lport - to-lport priority: type: integer minimum: 0 maximum: 32767 match: type: string action: type: string enum: - allow-related - allow-stateless - allow - drop - reject scope: Cluster names: plural: subnets singular: subnet kind: Subnet shortNames: - subnet --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: vlans.kubeovn.io spec: group: kubeovn.io versions: - name: v1 served: true storage: true subresources: status: {} schema: openAPIV3Schema: type: object properties: spec: type: object properties: id: type: integer minimum: 0 maximum: 4095 provider: type: string vlanId: type: integer description: Deprecated in favor of id providerInterfaceName: type: string description: Deprecated in favor of provider required: - provider status: type: object properties: subnets: type: array items: type: string additionalPrinterColumns: - name: ID type: string jsonPath: .spec.id - name: Provider type: string jsonPath: .spec.provider scope: Cluster names: plural: vlans singular: vlan kind: Vlan shortNames: - vlan --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: provider-networks.kubeovn.io spec: group: kubeovn.io versions: - name: v1 served: true storage: true subresources: status: {} schema: openAPIV3Schema: type: object properties: metadata: type: object properties: name: type: string maxLength: 12 not: enum: - int - external spec: type: object properties: defaultInterface: type: string maxLength: 15 pattern: '^[^/\s]+$' customInterfaces: type: array items: type: object properties: interface: type: string maxLength: 15 pattern: '^[^/\s]+$' nodes: type: array items: type: string exchangeLinkName: type: boolean excludeNodes: type: array items: type: string required: - defaultInterface status: type: object properties: ready: type: boolean readyNodes: type: array items: type: string vlans: type: array items: type: string conditions: type: array items: type: object properties: node: type: string type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string additionalPrinterColumns: - name: DefaultInterface type: string jsonPath: .spec.defaultInterface - name: Ready type: boolean jsonPath: .status.ready scope: Cluster names: plural: provider-networks singular: provider-network kind: ProviderNetwork listKind: ProviderNetworkList --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: security-groups.kubeovn.io spec: group: kubeovn.io names: plural: security-groups singular: security-group shortNames: - sg kind: SecurityGroup listKind: SecurityGroupList scope: Cluster versions: - name: v1 served: true storage: true schema: openAPIV3Schema: type: object properties: spec: type: object properties: ingressRules: type: array items: type: object properties: ipVersion: type: string protocol: type: string priority: type: integer remoteType: type: string remoteAddress: type: string remoteSecurityGroup: type: string portRangeMin: type: integer portRangeMax: type: integer policy: type: string egressRules: type: array items: type: object properties: ipVersion: type: string protocol: type: string priority: type: integer remoteType: type: string remoteAddress: type: string remoteSecurityGroup: type: string portRangeMin: type: integer portRangeMax: type: integer policy: type: string allowSameGroupTraffic: type: boolean status: type: object properties: portGroup: type: string allowSameGroupTraffic: type: boolean ingressMd5: type: string egressMd5: type: string ingressLastSyncSuccess: type: boolean egressLastSyncSuccess: type: boolean subresources: status: {} conversion: strategy: None --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: htbqoses.kubeovn.io spec: group: kubeovn.io versions: - name: v1 served: true storage: true additionalPrinterColumns: - name: PRIORITY type: string jsonPath: .spec.priority schema: openAPIV3Schema: type: object properties: spec: type: object properties: priority: type: string # Value in range 0 to 4,294,967,295. scope: Cluster names: plural: htbqoses singular: htbqos kind: HtbQos shortNames: - htbqos `))) OVN = template.Must(template.New("ovn.yaml").Parse( dedent.Dedent(`--- {{ if .DpdkMode }} apiVersion: v1 kind: ServiceAccount metadata: name: ovn namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: rbac.authorization.k8s.io/system-only: "true" name: system:ovn rules: - apiGroups: ['policy'] resources: ['podsecuritypolicies'] verbs: ['use'] resourceNames: - kube-ovn - apiGroups: - "kubeovn.io" resources: - vpcs - vpcs/status - vpc-nat-gateways - subnets - subnets/status - ips - vips - vips/status - vlans - vlans/status - provider-networks - provider-networks/status - security-groups - security-groups/status - htbqoses - iptables-eips - iptables-fip-rules - iptables-dnat-rules - iptables-snat-rules - iptables-eips/status - iptables-fip-rules/status - iptables-dnat-rules/status - iptables-snat-rules/status verbs: - "*" - apiGroups: - "" resources: - pods - pods/exec - namespaces - nodes - configmaps verbs: - create - get - list - watch - patch - update - apiGroups: - "k8s.cni.cncf.io" resources: - network-attachment-definitions verbs: - create - delete - get - list - update - apiGroups: - "" - networking.k8s.io - apps - extensions resources: - networkpolicies - services - endpoints - statefulsets - daemonsets - deployments - deployments/scale verbs: - create - delete - update - patch - get - list - watch - apiGroups: - "" resources: - events verbs: - create - patch - update - apiGroups: - coordination.k8s.io resources: - leases verbs: - "*" - apiGroups: - "k8s.cni.cncf.io" resources: - network-attachment-definitions verbs: - create - delete - get - list - update - apiGroups: - "kubevirt.io" resources: - virtualmachines - virtualmachineinstances verbs: - get - list --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: ovn roleRef: name: system:ovn kind: ClusterRole apiGroup: rbac.authorization.k8s.io subjects: - kind: ServiceAccount name: ovn namespace: kube-system --- kind: Service apiVersion: v1 metadata: name: ovn-nb namespace: kube-system spec: ports: - name: ovn-nb protocol: TCP port: 6641 targetPort: 6641 type: ClusterIP {{ .SvcYamlIpfamilypolicy }} selector: app: ovn-central ovn-nb-leader: "true" sessionAffinity: None --- kind: Service apiVersion: v1 metadata: name: ovn-sb namespace: kube-system spec: ports: - name: ovn-sb protocol: TCP port: 6642 targetPort: 6642 type: ClusterIP {{ .SvcYamlIpfamilypolicy }} selector: app: ovn-central ovn-sb-leader: "true" sessionAffinity: None --- kind: Service apiVersion: v1 metadata: name: ovn-northd namespace: kube-system spec: ports: - name: ovn-northd protocol: TCP port: 6643 targetPort: 6643 type: ClusterIP {{ .SvcYamlIpfamilypolicy }} selector: app: ovn-central ovn-northd-leader: "true" sessionAffinity: None --- kind: Deployment apiVersion: apps/v1 metadata: name: ovn-central namespace: kube-system annotations: kubernetes.io/description: | OVN components: northd, nb and sb. spec: replicas: {{ .Count }} strategy: rollingUpdate: maxSurge: 0 maxUnavailable: 1 type: RollingUpdate selector: matchLabels: app: ovn-central template: metadata: labels: app: ovn-central component: network type: infra spec: tolerations: - effect: NoSchedule operator: Exists - effect: NoExecute operator: Exists - key: CriticalAddonsOnly operator: Exists affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app: ovn-central topologyKey: kubernetes.io/hostname priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true containers: - name: ovn-central image: "{{ .KubeovnImage }}" imagePullPolicy: IfNotPresent command: ["/kube-ovn/start-db.sh"] securityContext: capabilities: add: ["SYS_NICE"] env: - name: ENABLE_SSL value: "{{ .EnableSSL }}" - name: NODE_IPS value: {{ .Address }} - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace resources: requests: cpu: 300m memory: 300Mi limits: cpu: 3 memory: 4Gi volumeMounts: - mountPath: /var/run/openvswitch name: host-run-ovs - mountPath: /var/run/ovn name: host-run-ovn - mountPath: /sys name: host-sys readOnly: true - mountPath: /etc/openvswitch name: host-config-openvswitch - mountPath: /etc/ovn name: host-config-ovn - mountPath: /var/log/openvswitch name: host-log-ovs - mountPath: /var/log/ovn name: host-log-ovn - mountPath: /etc/localtime name: localtime - mountPath: /var/run/tls name: kube-ovn-tls readinessProbe: exec: command: - bash - /kube-ovn/ovn-healthcheck.sh periodSeconds: 15 timeoutSeconds: 45 livenessProbe: exec: command: - bash - /kube-ovn/ovn-healthcheck.sh initialDelaySeconds: 30 periodSeconds: 15 failureThreshold: 5 timeoutSeconds: 45 nodeSelector: kubernetes.io/os: "linux" kube-ovn/role: "master" volumes: - name: host-run-ovs hostPath: path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - name: host-sys hostPath: path: /sys - name: host-config-openvswitch hostPath: path: /etc/origin/openvswitch - name: host-config-ovn hostPath: path: /etc/origin/ovn - name: host-log-ovs hostPath: path: /var/log/openvswitch - name: host-log-ovn hostPath: path: /var/log/ovn - name: localtime hostPath: path: /etc/localtime - name: kube-ovn-tls secret: optional: true secretName: kube-ovn-tls --- kind: DaemonSet apiVersion: apps/v1 metadata: name: ovs-ovn namespace: kube-system annotations: kubernetes.io/description: | This daemon set launches the openvswitch daemon. spec: selector: matchLabels: app: ovs updateStrategy: type: OnDelete template: metadata: labels: app: ovs component: network type: infra spec: tolerations: - effect: NoSchedule operator: Exists - effect: NoExecute operator: Exists - key: CriticalAddonsOnly operator: Exists priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true hostPID: true containers: - name: openvswitch image: "kubeovn/kube-ovn-dpdk:{{ .DpdkVersion }}-{{ .OvnVersion }}" imagePullPolicy: IfNotPresent command: ["/kube-ovn/start-ovs-dpdk.sh"] securityContext: runAsUser: 0 privileged: true env: - name: ENABLE_SSL value: "{{ .EnableSSL }}" - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: KUBE_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: OVN_DB_IPS value: {{ .Address }} volumeMounts: - mountPath: /var/run/netns name: host-ns mountPropagation: HostToContainer - mountPath: /lib/modules name: host-modules readOnly: true - mountPath: /var/run/openvswitch name: host-run-ovs - mountPath: /var/run/ovn name: host-run-ovn - mountPath: /sys name: host-sys readOnly: true - mountPath: /etc/cni/net.d name: cni-conf - mountPath: /etc/openvswitch name: host-config-openvswitch - mountPath: /etc/ovn name: host-config-ovn - mountPath: /var/log/openvswitch name: host-log-ovs - mountPath: /var/log/ovn name: host-log-ovn - mountPath: /opt/ovs-config name: host-config-ovs - mountPath: /dev/hugepages name: hugepage - mountPath: /etc/localtime name: localtime - mountPath: /var/run/tls name: kube-ovn-tls readinessProbe: exec: command: - bash - /kube-ovn/ovs-dpdk-healthcheck.sh periodSeconds: 5 timeoutSeconds: 45 livenessProbe: exec: command: - bash - /kube-ovn/ovs-dpdk-healthcheck.sh initialDelaySeconds: 60 periodSeconds: 5 failureThreshold: 5 timeoutSeconds: 45 resources: requests: cpu: 1000m memory: 2Gi limits: cpu: 1000m memory: 2Gi hugepages-1Gi: 1Gi nodeSelector: kubernetes.io/os: "linux" ovn.kubernetes.io/ovs_dp_type: "kernel" volumes: - name: host-modules hostPath: path: /lib/modules - name: host-run-ovs hostPath: path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - name: host-sys hostPath: path: /sys - name: host-ns hostPath: path: /var/run/netns - name: cni-conf hostPath: path: /etc/cni/net.d - name: host-config-openvswitch hostPath: path: /etc/origin/openvswitch - name: host-config-ovn hostPath: path: /etc/origin/ovn - name: host-log-ovs hostPath: path: /var/log/openvswitch - name: host-log-ovn hostPath: path: /var/log/ovn - name: host-config-ovs hostPath: path: /opt/ovs-config type: DirectoryOrCreate - name: hugepage emptyDir: medium: HugePages - name: localtime hostPath: path: /etc/localtime - name: kube-ovn-tls secret: optional: true secretName: kube-ovn-tls {{ else }} apiVersion: v1 kind: ServiceAccount metadata: name: ovn namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: rbac.authorization.k8s.io/system-only: "true" name: system:ovn rules: - apiGroups: ['policy'] resources: ['podsecuritypolicies'] verbs: ['use'] resourceNames: - kube-ovn - apiGroups: - "kubeovn.io" resources: - vpcs - vpcs/status - vpc-nat-gateways - subnets - subnets/status - ips - vips - vips/status - vlans - vlans/status - provider-networks - provider-networks/status - security-groups - security-groups/status - htbqoses - iptables-eips - iptables-fip-rules - iptables-dnat-rules - iptables-snat-rules - iptables-eips/status - iptables-fip-rules/status - iptables-dnat-rules/status - iptables-snat-rules/status verbs: - "*" - apiGroups: - "" resources: - pods - pods/exec - namespaces - nodes - configmaps verbs: - create - get - list - watch - patch - update - apiGroups: - "" - networking.k8s.io - apps - extensions resources: - networkpolicies - services - endpoints - statefulsets - daemonsets - deployments - deployments/scale verbs: - create - delete - update - patch - get - list - watch - apiGroups: - "" resources: - events verbs: - create - patch - update - apiGroups: - coordination.k8s.io resources: - leases verbs: - "*" - apiGroups: - "k8s.cni.cncf.io" resources: - network-attachment-definitions verbs: - create - delete - get - list - update - apiGroups: - "kubevirt.io" resources: - virtualmachines - virtualmachineinstances verbs: - get - list --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: ovn roleRef: name: system:ovn kind: ClusterRole apiGroup: rbac.authorization.k8s.io subjects: - kind: ServiceAccount name: ovn namespace: kube-system --- kind: Service apiVersion: v1 metadata: name: ovn-nb namespace: kube-system spec: ports: - name: ovn-nb protocol: TCP port: 6641 targetPort: 6641 type: ClusterIP {{ .SvcYamlIpfamilypolicy }} selector: app: ovn-central ovn-nb-leader: "true" sessionAffinity: None --- kind: Service apiVersion: v1 metadata: name: ovn-sb namespace: kube-system spec: ports: - name: ovn-sb protocol: TCP port: 6642 targetPort: 6642 type: ClusterIP {{ .SvcYamlIpfamilypolicy }} selector: app: ovn-central ovn-sb-leader: "true" sessionAffinity: None --- kind: Service apiVersion: v1 metadata: name: ovn-northd namespace: kube-system spec: ports: - name: ovn-northd protocol: TCP port: 6643 targetPort: 6643 type: ClusterIP {{ .SvcYamlIpfamilypolicy }} selector: app: ovn-central ovn-northd-leader: "true" sessionAffinity: None --- kind: Deployment apiVersion: apps/v1 metadata: name: ovn-central namespace: kube-system annotations: kubernetes.io/description: | OVN components: northd, nb and sb. spec: replicas: {{ .Count }} strategy: rollingUpdate: maxSurge: 0 maxUnavailable: 1 type: RollingUpdate selector: matchLabels: app: ovn-central template: metadata: labels: app: ovn-central component: network type: infra spec: tolerations: - effect: NoSchedule operator: Exists - effect: NoExecute operator: Exists - key: CriticalAddonsOnly operator: Exists affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app: ovn-central topologyKey: kubernetes.io/hostname priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true containers: - name: ovn-central image: "{{ .KubeovnImage }}" imagePullPolicy: IfNotPresent command: ["/kube-ovn/start-db.sh"] securityContext: capabilities: add: ["SYS_NICE"] env: - name: ENABLE_SSL value: "{{ .EnableSSL }}" - name: NODE_IPS value: {{ .Address }} - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace resources: requests: cpu: 300m memory: 200Mi limits: cpu: 3 memory: 4Gi volumeMounts: - mountPath: /var/run/openvswitch name: host-run-ovs - mountPath: /var/run/ovn name: host-run-ovn - mountPath: /sys name: host-sys readOnly: true - mountPath: /etc/openvswitch name: host-config-openvswitch - mountPath: /etc/ovn name: host-config-ovn - mountPath: /var/log/openvswitch name: host-log-ovs - mountPath: /var/log/ovn name: host-log-ovn - mountPath: /etc/localtime name: localtime - mountPath: /var/run/tls name: kube-ovn-tls readinessProbe: exec: command: - bash - /kube-ovn/ovn-healthcheck.sh periodSeconds: 15 timeoutSeconds: 45 livenessProbe: exec: command: - bash - /kube-ovn/ovn-healthcheck.sh initialDelaySeconds: 30 periodSeconds: 15 failureThreshold: 5 timeoutSeconds: 45 nodeSelector: kubernetes.io/os: "linux" kube-ovn/role: "master" volumes: - name: host-run-ovs hostPath: path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - name: host-sys hostPath: path: /sys - name: host-config-openvswitch hostPath: path: /etc/origin/openvswitch - name: host-config-ovn hostPath: path: /etc/origin/ovn - name: host-log-ovs hostPath: path: /var/log/openvswitch - name: host-log-ovn hostPath: path: /var/log/ovn - name: localtime hostPath: path: /etc/localtime - name: kube-ovn-tls secret: optional: true secretName: kube-ovn-tls --- kind: DaemonSet apiVersion: apps/v1 metadata: name: ovs-ovn namespace: kube-system annotations: kubernetes.io/description: | This daemon set launches the openvswitch daemon. spec: selector: matchLabels: app: ovs updateStrategy: type: OnDelete template: metadata: labels: app: ovs component: network type: infra spec: tolerations: - effect: NoSchedule operator: Exists - effect: NoExecute operator: Exists - key: CriticalAddonsOnly operator: Exists priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true hostPID: true containers: - name: openvswitch image: "{{ .KubeovnImage }}" imagePullPolicy: IfNotPresent command: ["/kube-ovn/start-ovs.sh"] securityContext: runAsUser: 0 privileged: true env: - name: ENABLE_SSL value: "{{ .EnableSSL }}" - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: HW_OFFLOAD value: "{{ .HwOffload }}" - name: TUNNEL_TYPE value: "{{ .TunnelType }}" - name: KUBE_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: OVN_DB_IPS value: {{ .Address }} volumeMounts: - mountPath: /var/run/netns name: host-ns mountPropagation: HostToContainer - mountPath: /lib/modules name: host-modules readOnly: true - mountPath: /var/run/openvswitch name: host-run-ovs - mountPath: /var/run/ovn name: host-run-ovn - mountPath: /sys name: host-sys readOnly: true - mountPath: /etc/cni/net.d name: cni-conf - mountPath: /etc/openvswitch name: host-config-openvswitch - mountPath: /etc/ovn name: host-config-ovn - mountPath: /var/log/openvswitch name: host-log-ovs - mountPath: /var/log/ovn name: host-log-ovn - mountPath: /etc/localtime name: localtime - mountPath: /var/run/tls name: kube-ovn-tls readinessProbe: exec: command: - bash - -c - LOG_ROTATE=true /kube-ovn/ovs-healthcheck.sh periodSeconds: 5 timeoutSeconds: 45 livenessProbe: exec: command: - bash - /kube-ovn/ovs-healthcheck.sh initialDelaySeconds: 60 periodSeconds: 5 failureThreshold: 5 timeoutSeconds: 45 resources: requests: cpu: 200m memory: 200Mi limits: cpu: 1000m memory: 1000Mi nodeSelector: kubernetes.io/os: "linux" volumes: - name: host-modules hostPath: path: /lib/modules - name: host-run-ovs hostPath: path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - name: host-sys hostPath: path: /sys - name: host-ns hostPath: path: /var/run/netns - name: cni-conf hostPath: path: /etc/cni/net.d - name: host-config-openvswitch hostPath: path: /etc/origin/openvswitch - name: host-config-ovn hostPath: path: /etc/origin/ovn - name: host-log-ovs hostPath: path: /var/log/openvswitch - name: host-log-ovn hostPath: path: /var/log/ovn - name: localtime hostPath: path: /etc/localtime - name: kube-ovn-tls secret: optional: true secretName: kube-ovn-tls {{ end }}`))) KubeOvn = template.Must(template.New("kube-ovn.yaml").Parse( dedent.Dedent(`--- kind: Deployment apiVersion: apps/v1 metadata: name: kube-ovn-controller namespace: kube-system annotations: kubernetes.io/description: | kube-ovn controller spec: replicas: {{ .Count }} selector: matchLabels: app: kube-ovn-controller strategy: rollingUpdate: maxSurge: 0% maxUnavailable: 100% type: RollingUpdate template: metadata: labels: app: kube-ovn-controller component: network type: infra spec: tolerations: - effect: NoSchedule operator: Exists - key: CriticalAddonsOnly operator: Exists affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app: kube-ovn-controller topologyKey: kubernetes.io/hostname priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true containers: - name: kube-ovn-controller image: "{{ .KubeovnImage }}" imagePullPolicy: IfNotPresent args: - /kube-ovn/start-controller.sh - --default-cidr={{ .PodCIDR }} - --default-gateway={{ .PodGateway }} - --default-gateway-check={{ .CheckGateway }} - --default-logical-gateway={{ .LogicalGateway }} - --default-exclude-ips={{ .ExcludeIps }} - --node-switch-cidr={{ .JoinCIDR }} - --service-cluster-ip-range={{ .SvcCIDR }} - --network-type={{ .NetworkType }} - --default-interface-name={{ .VlanInterfaceName }} - --default-vlan-id={{ .VlanID }} - --pod-nic-type={{ .PodNicType }} - --enable-lb={{ .EnableLB }} - --enable-np={{ .EnableNP }} - --enable-eip-snat={{ .EnableEipSnat }} - --enable-external-vpc={{ .EnableExternalVPC }} - --logtostderr=false - --alsologtostderr=true - --log_file=/var/log/kube-ovn/kube-ovn-controller.log - --log_file_max_size=0 - --keep-vm-ip=true env: - name: ENABLE_SSL value: "{{ .EnableSSL }}" - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: KUBE_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: KUBE_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: OVN_DB_IPS value: {{ .Address }} volumeMounts: - mountPath: /etc/localtime name: localtime - mountPath: /var/log/kube-ovn name: kube-ovn-log - mountPath: /var/run/tls name: kube-ovn-tls readinessProbe: exec: command: - /kube-ovn/kube-ovn-controller-healthcheck periodSeconds: 3 timeoutSeconds: 45 livenessProbe: exec: command: - /kube-ovn/kube-ovn-controller-healthcheck initialDelaySeconds: 300 periodSeconds: 7 failureThreshold: 5 timeoutSeconds: 45 resources: requests: cpu: 200m memory: 200Mi limits: cpu: 1000m memory: 1Gi nodeSelector: kubernetes.io/os: "linux" volumes: - name: localtime hostPath: path: /etc/localtime - name: kube-ovn-log hostPath: path: /var/log/kube-ovn - name: kube-ovn-tls secret: optional: true secretName: kube-ovn-tls --- kind: DaemonSet apiVersion: apps/v1 metadata: name: kube-ovn-cni namespace: kube-system annotations: kubernetes.io/description: | This daemon set launches the kube-ovn cni daemon. spec: selector: matchLabels: app: kube-ovn-cni template: metadata: labels: app: kube-ovn-cni component: network type: infra spec: tolerations: - effect: NoSchedule operator: Exists - effect: NoExecute operator: Exists - key: CriticalAddonsOnly operator: Exists priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true hostPID: true initContainers: - name: install-cni image: "{{ .KubeovnImage }}" imagePullPolicy: IfNotPresent command: ["/kube-ovn/install-cni.sh"] securityContext: runAsUser: 0 privileged: true volumeMounts: - mountPath: /opt/cni/bin name: cni-bin containers: - name: cni-server image: "{{ .KubeovnImage }}" imagePullPolicy: IfNotPresent command: - bash - /kube-ovn/start-cniserver.sh args: - --enable-mirror={{ .EnableMirror }} - --encap-checksum=true - --service-cluster-ip-range={{ .SvcCIDR }} - --iface={{ .Iface }} - --dpdk-tunnel-iface={{ .DpdkTunnelIface }} - --network-type={{ .TunnelType }} - --default-interface-name={{ .VlanInterfaceName }} - --cni-conf-name={{ .CNIConfigPriority }}-kube-ovn.conflist - --logtostderr=false - --alsologtostderr=true - --log_file=/var/log/kube-ovn/kube-ovn-cni.log - --log_file_max_size=0 securityContext: runAsUser: 0 privileged: true env: - name: ENABLE_SSL value: "{{ .EnableSSL }}" - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: KUBE_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: MODULES value: {{ .Modules }} - name: RPMS value: {{ .RPMs }} volumeMounts: - name: host-modules mountPath: /lib/modules readOnly: true - name: shared-dir mountPath: /var/lib/kubelet/pods - mountPath: /etc/openvswitch name: systemid - mountPath: /etc/cni/net.d name: cni-conf - mountPath: /run/openvswitch name: host-run-ovs mountPropagation: Bidirectional - mountPath: /run/ovn name: host-run-ovn - mountPath: /var/run/netns name: host-ns mountPropagation: HostToContainer - mountPath: /var/log/kube-ovn name: kube-ovn-log - mountPath: /var/log/openvswitch name: host-log-ovs - mountPath: /var/log/ovn name: host-log-ovn - mountPath: /etc/localtime name: localtime - mountPath: /tmp name: tmp livenessProbe: failureThreshold: 3 initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 tcpSocket: port: 10665 timeoutSeconds: 3 readinessProbe: failureThreshold: 3 initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 tcpSocket: port: 10665 timeoutSeconds: 3 resources: requests: cpu: 100m memory: 100Mi limits: cpu: 1000m memory: 1Gi nodeSelector: kubernetes.io/os: "linux" volumes: - name: host-modules hostPath: path: /lib/modules - name: shared-dir hostPath: path: /var/lib/kubelet/pods - name: systemid hostPath: path: /etc/origin/openvswitch - name: host-run-ovs hostPath: path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - name: cni-conf hostPath: path: /etc/cni/net.d - name: cni-bin hostPath: path: /opt/cni/bin - name: host-ns hostPath: path: /var/run/netns - name: host-log-ovs hostPath: path: /var/log/openvswitch - name: kube-ovn-log hostPath: path: /var/log/kube-ovn - name: host-log-ovn hostPath: path: /var/log/ovn - name: localtime hostPath: path: /etc/localtime - name: tmp hostPath: path: /tmp --- kind: DaemonSet apiVersion: apps/v1 metadata: name: kube-ovn-pinger namespace: kube-system annotations: kubernetes.io/description: | This daemon set launches the openvswitch daemon. spec: selector: matchLabels: app: kube-ovn-pinger updateStrategy: type: RollingUpdate template: metadata: labels: app: kube-ovn-pinger component: network type: infra spec: serviceAccountName: ovn hostPID: true containers: - name: pinger image: "{{ .KubeovnImage }}" command: - /kube-ovn/kube-ovn-pinger args: - --external-address={{ .PingExternalAddress }} - --external-dns={{ .PingExternalDNS }} - --logtostderr=false - --alsologtostderr=true - --log_file=/var/log/kube-ovn/kube-ovn-pinger.log - --log_file_max_size=0 imagePullPolicy: IfNotPresent securityContext: runAsUser: 0 privileged: false env: - name: ENABLE_SSL value: "{{ .EnableSSL }}" - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: HOST_IP valueFrom: fieldRef: fieldPath: status.hostIP - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName volumeMounts: - mountPath: /lib/modules name: host-modules readOnly: true - mountPath: /run/openvswitch name: host-run-ovs - mountPath: /var/run/openvswitch name: host-run-ovs - mountPath: /var/run/ovn name: host-run-ovn - mountPath: /sys name: host-sys readOnly: true - mountPath: /etc/openvswitch name: host-config-openvswitch - mountPath: /var/log/openvswitch name: host-log-ovs - mountPath: /var/log/ovn name: host-log-ovn - mountPath: /var/log/kube-ovn name: kube-ovn-log - mountPath: /etc/localtime name: localtime - mountPath: /var/run/tls name: kube-ovn-tls resources: requests: cpu: 100m memory: 100Mi limits: cpu: 200m memory: 400Mi nodeSelector: kubernetes.io/os: "linux" volumes: - name: host-modules hostPath: path: /lib/modules - name: host-run-ovs hostPath: path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - name: host-sys hostPath: path: /sys - name: host-config-openvswitch hostPath: path: /etc/origin/openvswitch - name: host-log-ovs hostPath: path: /var/log/openvswitch - name: kube-ovn-log hostPath: path: /var/log/kube-ovn - name: host-log-ovn hostPath: path: /var/log/ovn - name: localtime hostPath: path: /etc/localtime - name: kube-ovn-tls secret: optional: true secretName: kube-ovn-tls --- kind: Deployment apiVersion: apps/v1 metadata: name: kube-ovn-monitor namespace: kube-system annotations: kubernetes.io/description: | Metrics for OVN components: northd, nb and sb. spec: replicas: 1 strategy: rollingUpdate: maxSurge: 1 maxUnavailable: 1 type: RollingUpdate selector: matchLabels: app: kube-ovn-monitor template: metadata: labels: app: kube-ovn-monitor component: network type: infra spec: tolerations: - effect: NoSchedule operator: Exists - key: CriticalAddonsOnly operator: Exists affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app: kube-ovn-monitor topologyKey: kubernetes.io/hostname priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true containers: - name: kube-ovn-monitor image: "{{ .KubeovnImage }}" imagePullPolicy: IfNotPresent command: ["/kube-ovn/start-ovn-monitor.sh"] securityContext: runAsUser: 0 privileged: false env: - name: ENABLE_SSL value: "{{ .EnableSSL }}" - name: KUBE_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName resources: requests: cpu: 200m memory: 200Mi limits: cpu: 200m memory: 200Mi volumeMounts: - mountPath: /var/run/openvswitch name: host-run-ovs - mountPath: /var/run/ovn name: host-run-ovn - mountPath: /etc/openvswitch name: host-config-openvswitch - mountPath: /etc/ovn name: host-config-ovn - mountPath: /var/log/openvswitch name: host-log-ovs - mountPath: /var/log/ovn name: host-log-ovn - mountPath: /etc/localtime name: localtime - mountPath: /var/run/tls name: kube-ovn-tls readinessProbe: exec: command: - cat - /var/run/ovn/ovn-controller.pid periodSeconds: 10 timeoutSeconds: 45 livenessProbe: exec: command: - cat - /var/run/ovn/ovn-controller.pid initialDelaySeconds: 30 periodSeconds: 10 failureThreshold: 5 timeoutSeconds: 45 nodeSelector: kubernetes.io/os: "linux" kube-ovn/role: "master" volumes: - name: host-run-ovs hostPath: path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - name: host-config-openvswitch hostPath: path: /etc/origin/openvswitch - name: host-config-ovn hostPath: path: /etc/origin/ovn - name: host-log-ovs hostPath: path: /var/log/openvswitch - name: host-log-ovn hostPath: path: /var/log/ovn - name: localtime hostPath: path: /etc/localtime - name: kube-ovn-tls secret: optional: true secretName: kube-ovn-tls --- kind: Service apiVersion: v1 metadata: name: kube-ovn-monitor namespace: kube-system labels: app: kube-ovn-monitor spec: ports: - name: metrics port: 10661 type: ClusterIP {{ .SvcYamlIpfamilypolicy }} selector: app: kube-ovn-monitor sessionAffinity: None --- kind: Service apiVersion: v1 metadata: name: kube-ovn-pinger namespace: kube-system labels: app: kube-ovn-pinger spec: {{ .SvcYamlIpfamilypolicy }} selector: app: kube-ovn-pinger ports: - port: 8080 name: metrics --- kind: Service apiVersion: v1 metadata: name: kube-ovn-controller namespace: kube-system labels: app: kube-ovn-controller spec: {{ .SvcYamlIpfamilypolicy }} selector: app: kube-ovn-controller ports: - port: 10660 name: metrics --- kind: Service apiVersion: v1 metadata: name: kube-ovn-cni namespace: kube-system labels: app: kube-ovn-cni spec: {{ .SvcYamlIpfamilypolicy }} selector: app: kube-ovn-cni ports: - port: 10665 name: metrics`))) )
View Source
var Cilium = template.Must(template.New("network-plugin.yaml").Parse( dedent.Dedent(`--- # Source: cilium/charts/agent/templates/serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: name: cilium namespace: kube-system --- # Source: cilium/charts/operator/templates/serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: name: cilium-operator namespace: kube-system --- # Source: cilium/charts/config/templates/configmap.yaml apiVersion: v1 kind: ConfigMap metadata: name: cilium-config namespace: kube-system data: # Identity allocation mode selects how identities are shared between cilium # nodes by setting how they are stored. The options are "crd" or "kvstore". # - "crd" stores identities in kubernetes as CRDs (custom resource definition). # These can be queried with: # kubectl get ciliumid # - "kvstore" stores identities in a kvstore, etcd or consul, that is # configured below. Cilium versions before 1.6 supported only the kvstore # backend. Upgrades from these older cilium versions should continue using # the kvstore by commenting out the identity-allocation-mode below, or # setting it to "kvstore". identity-allocation-mode: crd # If you want to run cilium in debug mode change this value to true debug: "false" # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 # address. enable-ipv4: "true" # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 # address. enable-ipv6: "false" enable-bpf-clock-probe: "true" # If you want cilium monitor to aggregate tracing for packets, set this level # to "low", "medium", or "maximum". The higher the level, the less packets # that will be seen in monitor output. monitor-aggregation: medium # The monitor aggregation interval governs the typical time between monitor # notification events for each allowed connection. # # Only effective when monitor aggregation is set to "medium" or higher. monitor-aggregation-interval: 5s # The monitor aggregation flags determine which TCP flags which, upon the # first observation, cause monitor notifications to be generated. # # Only effective when monitor aggregation is set to "medium" or higher. monitor-aggregation-flags: all # bpf-policy-map-max specified the maximum number of entries in endpoint # policy map (per endpoint) bpf-policy-map-max: "16384" # Specifies the ratio (0.0-1.0) of total system memory to use for dynamic # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. bpf-map-dynamic-size-ratio: "0.0025" # Pre-allocation of map entries allows per-packet latency to be reduced, at # the expense of up-front memory allocation for the entries in the maps. The # default value below will minimize memory usage in the default installation; # users who are sensitive to latency may consider setting this to "true". # # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore # this option and behave as though it is set to "true". # # If this value is modified, then during the next Cilium startup the restore # of existing endpoints and tracking of ongoing connections may be disrupted. # This may lead to policy drops or a change in loadbalancing decisions for a # connection for some time. Endpoints may need to be recreated to restore # connectivity. # # If this option is set to "false" during an upgrade from 1.3 or earlier to # 1.4 or later, then it may cause one-time disruptions during the upgrade. preallocate-bpf-maps: "false" # Regular expression matching compatible Istio sidecar istio-proxy # container image names sidecar-istio-proxy-image: "cilium/istio_proxy" # Encapsulation mode for communication between nodes # Possible values: # - disabled # - vxlan (default) # - geneve tunnel: vxlan # Name of the cluster. Only relevant when building a mesh of clusters. cluster-name: default # wait-bpf-mount makes init container wait until bpf filesystem is mounted wait-bpf-mount: "false" masquerade: "true" enable-bpf-masquerade: "true" enable-xt-socket-fallback: "true" install-iptables-rules: "true" auto-direct-node-routes: "false" kube-proxy-replacement: "probe" enable-health-check-nodeport: "true" node-port-bind-protection: "true" enable-auto-protect-node-port-range: "true" enable-session-affinity: "true" k8s-require-ipv4-pod-cidr: "true" k8s-require-ipv6-pod-cidr: "false" enable-endpoint-health-checking: "true" enable-well-known-identities: "false" enable-remote-node-identity: "true" operator-api-serve-addr: "127.0.0.1:9234" ipam: "cluster-pool" cluster-pool-ipv4-cidr: "{{ .KubePodsCIDR }}" cluster-pool-ipv4-mask-size: "{{ .NodeCidrMaskSize }}" disable-cnp-status-updates: "true" --- # Source: cilium/charts/agent/templates/clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cilium rules: - apiGroups: - networking.k8s.io resources: - networkpolicies verbs: - get - list - watch - apiGroups: - discovery.k8s.io resources: - endpointslices verbs: - get - list - watch - apiGroups: - "" resources: - namespaces - services - nodes - endpoints verbs: - get - list - watch - apiGroups: - "" resources: - pods - nodes verbs: - get - list - watch - update - apiGroups: - "" resources: - nodes - nodes/status verbs: - patch - apiGroups: - apiextensions.k8s.io resources: - customresourcedefinitions verbs: - create - get - list - watch - update - apiGroups: - cilium.io resources: - ciliumnetworkpolicies - ciliumnetworkpolicies/status - ciliumclusterwidenetworkpolicies - ciliumclusterwidenetworkpolicies/status - ciliumendpoints - ciliumendpoints/status - ciliumnodes - ciliumnodes/status - ciliumidentities # deprecated remove in v1.9 - ciliumidentities/status verbs: - '*' --- # Source: cilium/charts/operator/templates/clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cilium-operator rules: - apiGroups: - "" resources: # to automatically delete [core|kube]dns pods so that are starting to being # managed by Cilium - pods verbs: - get - list - watch - delete - apiGroups: - discovery.k8s.io resources: - endpointslices verbs: - get - list - watch - apiGroups: - "" resources: # to perform the translation of a CNP that contains ToGroup to its endpoints - services - endpoints # to check apiserver connectivity - namespaces verbs: - get - list - watch - apiGroups: - cilium.io resources: - ciliumnetworkpolicies - ciliumnetworkpolicies/status - ciliumclusterwidenetworkpolicies - ciliumclusterwidenetworkpolicies/status - ciliumendpoints - ciliumendpoints/status - ciliumnodes - ciliumnodes/status - ciliumidentities - ciliumidentities/status verbs: - '*' - apiGroups: - apiextensions.k8s.io resources: - customresourcedefinitions verbs: - get - list - watch # For cilium-operator running in HA mode. # # Cilium operator running in HA mode requires the use of ResourceLock for Leader Election # between mulitple running instances. # The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less # common and fewer objects in the cluster watch "all Leases". # The support for leases was introduced in coordination.k8s.io/v1 during Kubernetes 1.14 release. # In Cilium we currently don't support HA mode for K8s version < 1.14. This condition make sure # that we only authorize access to leases resources in supported K8s versions. - apiGroups: - coordination.k8s.io resources: - leases verbs: - create - get - update --- # Source: cilium/charts/agent/templates/clusterrolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cilium roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cilium subjects: - kind: ServiceAccount name: cilium namespace: kube-system --- # Source: cilium/charts/operator/templates/clusterrolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cilium-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cilium-operator subjects: - kind: ServiceAccount name: cilium-operator namespace: kube-system --- # Source: cilium/charts/agent/templates/daemonset.yaml apiVersion: apps/v1 kind: DaemonSet metadata: labels: k8s-app: cilium name: cilium namespace: kube-system spec: selector: matchLabels: k8s-app: cilium template: metadata: annotations: # This annotation plus the CriticalAddonsOnly toleration makes # cilium to be a critical pod in the cluster, which ensures cilium # gets priority scheduling. # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ scheduler.alpha.kubernetes.io/critical-pod: "" labels: k8s-app: cilium spec: affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - podAffinityTerm: labelSelector: matchExpressions: - key: io.cilium/app operator: In values: - operator topologyKey: kubernetes.io/hostname weight: 100 containers: - args: - --config-dir=/tmp/cilium/config-map command: - cilium-agent livenessProbe: httpGet: host: '127.0.0.1' path: /healthz port: 9876 scheme: HTTP httpHeaders: - name: "brief" value: "true" failureThreshold: 10 # The initial delay for the liveness probe is intentionally large to # avoid an endless kill & restart cycle if in the event that the initial # bootstrapping takes longer than expected. initialDelaySeconds: 120 periodSeconds: 30 successThreshold: 1 timeoutSeconds: 5 readinessProbe: httpGet: host: '127.0.0.1' path: /healthz port: 9876 scheme: HTTP httpHeaders: - name: "brief" value: "true" failureThreshold: 3 initialDelaySeconds: 5 periodSeconds: 30 successThreshold: 1 timeoutSeconds: 5 env: - name: K8S_NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - name: CILIUM_K8S_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - name: CILIUM_FLANNEL_MASTER_DEVICE valueFrom: configMapKeyRef: key: flannel-master-device name: cilium-config optional: true - name: CILIUM_FLANNEL_UNINSTALL_ON_EXIT valueFrom: configMapKeyRef: key: flannel-uninstall-on-exit name: cilium-config optional: true - name: CILIUM_CLUSTERMESH_CONFIG value: /var/lib/cilium/clustermesh/ - name: CILIUM_CNI_CHAINING_MODE valueFrom: configMapKeyRef: key: cni-chaining-mode name: cilium-con optional: true - name: CILIUM_CUSTOM_CNI_CONF valueFrom: configMapKeyRef: key: custom-cni-conf name: cilium-config optional: true image: "{{ .CiliumImage }}" imagePullPolicy: IfNotPresent lifecycle: postStart: exec: command: - "/cni-install.sh" - "--enable-debug=false" preStop: exec: command: - /cni-uninstall.sh name: cilium-agent securityContext: capabilities: add: - NET_ADMIN - SYS_MODULE privileged: true volumeMounts: - mountPath: /sys/fs/bpf name: bpf-maps - mountPath: /var/run/cilium name: cilium-run - mountPath: /host/opt/cni/bin name: cni-path - mountPath: /host/etc/cni/net.d name: etc-cni-netd - mountPath: /var/lib/cilium/clustermesh name: clustermesh-secrets readOnly: true - mountPath: /tmp/cilium/config-map name: cilium-config-path readOnly: true # Needed to be able to load kernel modules - mountPath: /lib/modules name: lib-modules readOnly: true - mountPath: /run/xtables.lock name: xtables-lock hostNetwork: true initContainers: - command: - /init-container.sh env: - name: CILIUM_ALL_STATE valueFrom: configMapKeyRef: key: clean-cilium-state name: cilium-config optional: true - name: CILIUM_BPF_STATE valueFrom: configMapKeyRef: key: clean-cilium-bpf-state name: cilium-config optional: true - name: CILIUM_WAIT_BPF_MOUNT valueFrom: configMapKeyRef: key: wait-bpf-mount name: cilium-config optional: true image: "{{ .CiliumImage }}" imagePullPolicy: IfNotPresent name: clean-cilium-state securityContext: capabilities: add: - NET_ADMIN privileged: true volumeMounts: - mountPath: /sys/fs/bpf name: bpf-maps mountPropagation: HostToContainer - mountPath: /var/run/cilium name: cilium-run resources: requests: cpu: 100m memory: 100Mi restartPolicy: Always priorityClassName: system-node-critical serviceAccount: cilium serviceAccountName: cilium terminationGracePeriodSeconds: 1 tolerations: - operator: Exists volumes: # To keep state between restarts / upgrades - hostPath: path: /var/run/cilium type: DirectoryOrCreate name: cilium-run # To keep state between restarts / upgrades for bpf maps - hostPath: path: /sys/fs/bpf type: DirectoryOrCreate name: bpf-maps # To install cilium cni plugin in the host - hostPath: path: /opt/cni/bin type: DirectoryOrCreate name: cni-path # To install cilium cni configuration in the host - hostPath: path: /etc/cni/net.d type: DirectoryOrCreate name: etc-cni-netd # To be able to load kernel modules - hostPath: path: /lib/modules name: lib-modules # To access iptables concurrently with other processes (e.g. kube-proxy) - hostPath: path: /run/xtables.lock type: FileOrCreate name: xtables-lock # To read the clustermesh configuration - name: clustermesh-secrets secret: defaultMode: 420 optional: true secretName: cilium-clustermesh # To read the configuration from the config map - configMap: name: cilium-config name: cilium-config-path updateStrategy: rollingUpdate: maxUnavailable: 2 type: RollingUpdate --- # Source: cilium/charts/operator/templates/deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: labels: io.cilium/app: operator name: cilium-operator name: cilium-operator namespace: kube-system spec: # We support HA mode only for Kubernetes version > 1.14 # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go # for more details. replicas: 2 selector: matchLabels: io.cilium/app: operator name: cilium-operator strategy: rollingUpdate: maxSurge: 1 maxUnavailable: 1 type: RollingUpdate template: metadata: annotations: labels: io.cilium/app: operator name: cilium-operator spec: # In HA mode, cilium-operator pods must not be scheduled on the same # node as they will clash with each other. affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - podAffinityTerm: labelSelector: matchExpressions: - key: io.cilium/app operator: In values: - operator topologyKey: kubernetes.io/hostname weight: 100 containers: - args: - --config-dir=/tmp/cilium/config-map - --debug=$(CILIUM_DEBUG) command: - cilium-operator-generic env: - name: K8S_NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - name: CILIUM_K8S_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - name: CILIUM_DEBUG valueFrom: configMapKeyRef: key: debug name: cilium-config optional: true - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: key: AWS_ACCESS_KEY_ID name: cilium-aws optional: true - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: key: AWS_SECRET_ACCESS_KEY name: cilium-aws optional: true - name: AWS_DEFAULT_REGION valueFrom: secretKeyRef: key: AWS_DEFAULT_REGION name: cilium-aws optional: true image: "{{ .OperatorGenericImage }}" imagePullPolicy: IfNotPresent name: cilium-operator livenessProbe: httpGet: host: '127.0.0.1' path: /healthz port: 9234 scheme: HTTP initialDelaySeconds: 60 periodSeconds: 10 timeoutSeconds: 3 volumeMounts: - mountPath: /tmp/cilium/config-map name: cilium-config-path readOnly: true hostNetwork: true restartPolicy: Always priorityClassName: system-cluster-critical serviceAccount: cilium-operator serviceAccountName: cilium-operator tolerations: - operator: Exists volumes: # To read the configuration from the config map - configMap: name: cilium-config name: cilium-config-path `)))
View Source
var FlannelPS = template.Must(template.New("network-plugin.yaml").Parse( dedent.Dedent(`--- apiVersion: v1 kind: Namespace metadata: name: kube-flannel labels: pod-security.kubernetes.io/enforce: privileged --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: labels: k8s-app: flannel name: flannel rules: - apiGroups: - "" resources: - pods verbs: - get - apiGroups: - "" resources: - nodes verbs: - get - list - watch - apiGroups: - "" resources: - nodes/status verbs: - patch - apiGroups: - networking.k8s.io resources: - clustercidrs verbs: - list - watch --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: labels: k8s-app: flannel name: flannel roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: flannel subjects: - kind: ServiceAccount name: flannel namespace: kube-flannel --- apiVersion: v1 kind: ServiceAccount metadata: name: flannel namespace: kube-flannel --- kind: ConfigMap apiVersion: v1 metadata: name: kube-flannel-cfg namespace: kube-flannel labels: tier: node app: flannel k8s-app: flannel data: cni-conf.json: | { "name": "cbr0", "cniVersion": "0.3.1", "plugins": [ { "type": "flannel", "delegate": { "hairpinMode": true, "isDefaultGateway": true } }, { "type": "portmap", "capabilities": { "portMappings": true } } ] } net-conf.json: | { "Network": "{{ .KubePodsCIDR }}", "Backend": { "Type": "{{ .BackendMode }}" } } --- apiVersion: apps/v1 kind: DaemonSet metadata: name: kube-flannel-ds namespace: kube-flannel labels: tier: node app: flannel k8s-app: flannel spec: selector: matchLabels: app: flannel k8s-app: flannel template: metadata: labels: tier: node app: flannel k8s-app: flannel spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/os operator: In values: - linux hostNetwork: true tolerations: - operator: Exists effect: NoSchedule priorityClassName: system-node-critical serviceAccountName: flannel initContainers: - name: install-cni-plugin args: - -f - /flannel - /opt/cni/bin/flannel command: - cp image: {{ .FlannelPluginImage }} volumeMounts: - mountPath: /opt/cni/bin name: cni-plugin - name: install-cni image: {{ .FlannelImage }} command: - cp args: - -f - /etc/kube-flannel/cni-conf.json - /etc/cni/net.d/10-flannel.conflist volumeMounts: - name: cni mountPath: /etc/cni/net.d - name: flannel-cfg mountPath: /etc/kube-flannel/ containers: - name: kube-flannel image: {{ .FlannelImage }} command: - /opt/bin/flanneld args: - --ip-masq - --kube-subnet-mgr resources: requests: cpu: "100m" memory: "50Mi" limits: cpu: "100m" memory: "50Mi" securityContext: privileged: false capabilities: add: ["NET_ADMIN", "NET_RAW"] env: - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace volumeMounts: - name: run mountPath: /run/flannel - name: flannel-cfg mountPath: /etc/kube-flannel/ - mountPath: /run/xtables.lock name: xtables-lock volumes: - name: run hostPath: path: /run/flannel - name: cni-plugin hostPath: path: /opt/cni/bin - name: cni hostPath: path: /etc/cni/net.d - name: xtables-lock hostPath: path: /run/xtables.lock type: FileOrCreate - name: flannel-cfg configMap: name: kube-flannel-cfg `)))
View Source
var FlannelPSP = template.Must(template.New("network-plugin.yaml").Parse( dedent.Dedent(`--- apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: name: psp.flannel.unprivileged annotations: seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default spec: privileged: false volumes: - configMap - secret - emptyDir - hostPath allowedHostPaths: - pathPrefix: "/etc/cni/net.d" - pathPrefix: "/etc/kube-flannel" - pathPrefix: "/run/flannel" readOnlyRootFilesystem: false # Users and groups runAsUser: rule: RunAsAny supplementalGroups: rule: RunAsAny fsGroup: rule: RunAsAny # Privilege Escalation allowPrivilegeEscalation: false defaultAllowPrivilegeEscalation: false # Capabilities allowedCapabilities: ['NET_ADMIN'] defaultAddCapabilities: [] requiredDropCapabilities: [] # Host namespaces hostPID: false hostIPC: false hostNetwork: true hostPorts: - min: 0 max: 65535 # SELinux seLinux: # SELinux is unused in CaaSP rule: 'RunAsAny' --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: labels: k8s-app: flannel name: flannel rules: - apiGroups: ['extensions'] resources: ['podsecuritypolicies'] verbs: ['use'] resourceNames: ['psp.flannel.unprivileged'] - apiGroups: - "" resources: - pods verbs: - get - apiGroups: - "" resources: - nodes verbs: - get - list - watch - apiGroups: - "" resources: - nodes/status verbs: - patch - apiGroups: - networking.k8s.io resources: - clustercidrs verbs: - list - watch --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: labels: k8s-app: flannel name: flannel roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: flannel subjects: - kind: ServiceAccount name: flannel namespace: kube-system --- apiVersion: v1 kind: ServiceAccount metadata: name: flannel namespace: kube-system --- kind: ConfigMap apiVersion: v1 metadata: name: kube-flannel-cfg namespace: kube-system labels: tier: node app: flannel k8s-app: flannel data: cni-conf.json: | { "name": "cbr0", "cniVersion": "0.3.1", "plugins": [ { "type": "flannel", "delegate": { "hairpinMode": true, "isDefaultGateway": true } }, { "type": "portmap", "capabilities": { "portMappings": true } } ] } net-conf.json: | { "Network": "{{ .KubePodsCIDR }}", "Backend": { "Type": "{{ .BackendMode }}" } } --- apiVersion: apps/v1 kind: DaemonSet metadata: name: kube-flannel-ds namespace: kube-system labels: tier: node app: flannel k8s-app: flannel spec: selector: matchLabels: app: flannel k8s-app: flannel template: metadata: labels: tier: node app: flannel k8s-app: flannel spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/os operator: In values: - linux hostNetwork: true tolerations: - operator: Exists effect: NoSchedule priorityClassName: system-node-critical serviceAccountName: flannel initContainers: - name: install-cni-plugin args: - -f - /flannel - /opt/cni/bin/flannel command: - cp image: {{ .FlannelPluginImage }} volumeMounts: - mountPath: /opt/cni/bin name: cni-plugin - name: install-cni image: {{ .FlannelImage }} command: - cp args: - -f - /etc/kube-flannel/cni-conf.json - /etc/cni/net.d/10-flannel.conflist volumeMounts: - name: cni mountPath: /etc/cni/net.d - name: flannel-cfg mountPath: /etc/kube-flannel/ containers: - name: kube-flannel image: {{ .FlannelImage }} command: - /opt/bin/flanneld args: - --ip-masq - --kube-subnet-mgr resources: requests: cpu: "100m" memory: "50Mi" limits: cpu: "100m" memory: "50Mi" securityContext: privileged: false capabilities: add: ["NET_ADMIN", "NET_RAW"] env: - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace volumeMounts: - name: run mountPath: /run/flannel - name: flannel-cfg mountPath: /etc/kube-flannel/ - mountPath: /run/xtables.lock name: xtables-lock volumes: - name: run hostPath: path: /run/flannel - name: cni-plugin hostPath: path: /opt/cni/bin - name: cni hostPath: path: /etc/cni/net.d - name: xtables-lock hostPath: path: /run/xtables.lock type: FileOrCreate - name: flannel-cfg configMap: name: kube-flannel-cfg `)))
View Source
var HybridnetNetworks = template.Must(template.New("hybridnet-networks.yaml").Funcs(utils.FuncMap).Parse( dedent.Dedent(` {{- range $index, $network := .Networks }} --- apiVersion: networking.alibaba.com/v1 kind: Network metadata: name: {{ $network.Name }} spec: {{- if $network.NetID }} netID: {{ $network.NetID }} {{- end }} type: {{ $network.Type }} {{- if $network.Mode }} mode: {{ $network.Mode }} {{- end }} {{- if $network.NodeSelector }} nodeSelector: {{ toYaml $network.NodeSelector | indent 4 }} {{- end }} {{- range $network.Subnets }} --- apiVersion: networking.alibaba.com/v1 kind: Subnet metadata: name: {{ .Name }} spec: network: {{ $network.Name }} {{- if .NetID }} netID: {{ .NetID }} {{- end }} range: version: "4" cidr: "{{ .CIDR }}" {{- if .Gateway }} gateway: "{{ .Gateway }}" {{- end }} {{- if .Start}} start: "{{ .Start }}" {{- end}} {{- if .End}} end: "{{ .End }}" {{- end }} {{- if .ReservedIPs }} reservedIPs: {{ toYaml .ReservedIPs | indent 4 }} {{- end }} {{- if .ExcludeIPs }} excludeIPs: {{ toYaml .ExcludeIPs | indent 4 }} {{- end }} {{- end }} {{- end }} `)))
View Source
var KubectlKo = template.Must(template.New("kubectl-ko").Parse( dedent.Dedent(`#!/bin/bash set -euo pipefail KUBE_OVN_NS=kube-system WITHOUT_KUBE_PROXY=false OVN_NB_POD= OVN_SB_POD= KUBE_OVN_VERSION= REGISTRY="kubeovn" showHelp(){ echo "kubectl ko {subcommand} [option...]" echo "Available Subcommands:" echo " [nb|sb] [status|kick|backup|dbstatus|restore] ovn-db operations show cluster status, kick stale server, backup database, get db consistency status or restore ovn nb db when met 'inconsistent data' error" echo " nbctl [ovn-nbctl options ...] invoke ovn-nbctl" echo " sbctl [ovn-sbctl options ...] invoke ovn-sbctl" echo " vsctl {nodeName} [ovs-vsctl options ...] invoke ovs-vsctl on the specified node" echo " ofctl {nodeName} [ovs-ofctl options ...] invoke ovs-ofctl on the specified node" echo " dpctl {nodeName} [ovs-dpctl options ...] invoke ovs-dpctl on the specified node" echo " appctl {nodeName} [ovs-appctl options ...] invoke ovs-appctl on the specified node" echo " tcpdump {namespace/podname} [tcpdump options ...] capture pod traffic" echo " trace {namespace/podname} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp or udp port] trace ovn microflow of specific packet" echo " diagnose {all|node} [nodename] diagnose connectivity of all nodes or a specific node" echo " tuning {install-fastpath|local-install-fastpath|remove-fastpath|install-stt|local-install-stt|remove-stt} {centos7|centos8}} [kernel-devel-version] deploy kernel optimisation components to the system" echo " reload restart all kube-ovn components" echo " env-check check the environment configuration" } # usage: ipv4_to_hex 192.168.0.1 ipv4_to_hex(){ printf "%02x" ${1//./ } } # convert hex to dec (portable version) hex2dec(){ for i in $(echo "$@"); do printf "%d\n" "$(( 0x$i ))" done } # https://github.com/chmduquesne/wg-ip # usage: expand_ipv6 2001::1 expand_ipv6(){ local ip=$1 # prepend 0 if we start with : echo $ip | grep -qs "^:" && ip="0${ip}" # expand :: if echo $ip | grep -qs "::"; then local colons=$(echo $ip | sed 's/[^:]//g') local missing=$(echo ":::::::::" | sed "s/$colons//") local expanded=$(echo $missing | sed 's/:/:0/g') ip=$(echo $ip | sed "s/::/$expanded/") fi local blocks=$(echo $ip | grep -o "[0-9a-f]\+") set $blocks printf "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n" \ $(hex2dec $@) } # convert an IPv6 address to bytes ipv6_bytes(){ for x in $(expand_ipv6 $1 | tr ':' ' '); do printf "%d %d " $((0x$x >> 8 & 0xff)) $((0x$x & 0xff)) done echo } # usage: ipIsInCidr 192.168.0.1 192.168.0.0/24 # return: 0 for true, 1 for false ipIsInCidr(){ local ip=$1 local cidr=$2 if [[ $ip =~ .*:.* ]]; then # IPv6 cidr=${cidr#*,} local network=${cidr%/*} local prefix=${cidr#*/} local ip_bytes=($(ipv6_bytes $ip)) local network_bytes=($(ipv6_bytes $network)) for ((i=0; i<${#ip_bytes[*]}; i++)); do if [ ${ip_bytes[$i]} -eq ${network_bytes[$i]} ]; then continue fi if [ $((($i+1)*8)) -le $prefix ]; then return 1 fi if [ $(($i*8)) -ge $prefix ]; then return 0 fi if [ $((($i+1)*8)) -le $prefix ]; then return 1 fi local bits=$(($prefix-$i*8)) local mask=$((0xff<<$bits & 0xff)) # TODO: check whether the IP is network/broadcast address if [ $((${ip_bytes[$i]} & $mask)) -ne ${network_bytes[$i]} ]; then return 1 fi done return 0 fi # IPv4 cidr=${cidr%,*} local network=${cidr%/*} local prefix=${cidr#*/} local ip_hex=$(ipv4_to_hex $ip) local ip_dec=$((0x$ip_hex)) local network_hex=$(ipv4_to_hex $network) local network_dec=$((0x$network_hex)) local broadcast_dec=$(($network_dec + 2**(32-$prefix) - 1)) # TODO: check whether the IP is network/broadcast address if [ $ip_dec -gt $network_dec -a $ip_dec -lt $broadcast_dec ]; then return 0 fi return 1 } tcpdump(){ namespacedPod="$1"; shift namespace=$(echo "$namespacedPod" | cut -d "/" -f1) podName=$(echo "$namespacedPod" | cut -d "/" -f2) if [ "$podName" = "$namespacedPod" ]; then namespace="default" fi nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName}) hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork}) if [ -z "$nodeName" ]; then echo "Pod $namespacedPod not exists on any node" exit 1 fi ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -l app=kube-ovn-cni -o 'jsonpath={.items[?(@.spec.nodeName=="'$nodeName'")].metadata.name}') if [ -z "$ovnCni" ]; then echo "kube-ovn-cni not exist on node $nodeName" exit 1 fi if [ "$hostNetwork" = "true" ]; then set -x kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- tcpdump -nn "$@" else nicName=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface external-ids:iface-id="$podName"."$namespace" | tr -d '\r') if [ -z "$nicName" ]; then echo "nic doesn't exist on node $nodeName" exit 1 fi podNicType=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type}) podNetNs=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r' | sed -e 's/^"//' -e 's/"$//') set -x if [ "$podNicType" = "internal-port" ]; then kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" tcpdump -nn -i "$nicName" "$@" else kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" tcpdump -nn -i eth0 "$@" fi fi } trace(){ namespacedPod="$1" namespace=$(echo "$namespacedPod" | cut -d "/" -f1) podName=$(echo "$namespacedPod" | cut -d "/" -f2) if [ "$podName" = "$namespacedPod" ]; then namespace="default" fi dst="$2" if [ -z "$dst" ]; then echo "need a target ip address" exit 1 fi hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork}) if [ "$hostNetwork" = "true" ]; then echo "Can not trace host network pod" exit 1 fi af="4" nw="nw" proto="" if [[ "$dst" =~ .*:.* ]]; then af="6" nw="ipv6" proto="6" fi podIPs=($(kubectl get pod "$podName" -n "$namespace" -o jsonpath="{.status.podIPs[*].ip}")) if [ ${#podIPs[@]} -eq 0 ]; then podIPs=($(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/ip_address} | sed 's/,/ /g')) if [ ${#podIPs[@]} -eq 0 ]; then echo "pod address not ready" exit 1 fi fi podIP="" for ip in ${podIPs[@]}; do if [ "$af" = "4" ]; then if [[ ! "$ip" =~ .*:.* ]]; then podIP=$ip break fi elif [[ "$ip" =~ .*:.* ]]; then podIP=$ip break fi done if [ -z "$podIP" ]; then echo "Pod $namespacedPod has no IPv$af address" exit 1 fi nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName}) ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -l app=kube-ovn-cni -o 'jsonpath={.items[?(@.spec.nodeName=="'$nodeName'")].metadata.name}') if [ -z "$ovnCni" ]; then echo "No kube-ovn-cni Pod running on node $nodeName" exit 1 fi ls=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_switch}) if [ -z "$ls" ]; then echo "pod address not ready" exit 1 fi local cidr=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/cidr}) mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address}) dstMac="" if echo "$3" | grep -qE '^([[:xdigit:]]{1,2}:){5}[[:xdigit:]]{1,2}$'; then dstMac=$3 shift elif ipIsInCidr $dst $cidr; then set +o pipefail if [ $af -eq 4 ]; then dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=addresses list logical_switch_port | grep -w "$(echo $dst | tr . '\.')" | awk '{print $1}') else dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=addresses list logical_switch_port | grep -i " $dst\$" | awk '{print $1}') fi set -o pipefail fi if [ -z "$dstMac" ]; then vlan=$(kubectl get subnet "$ls" -o jsonpath={.spec.vlan}) logicalGateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.logicalGateway}) if [ ! -z "$vlan" -a "$logicalGateway" != "true" ]; then gateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.gateway}) if [[ "$gateway" =~ .*,.* ]]; then if [ "$af" = "4" ]; then gateway=${gateway%%,*} else gateway=${gateway##*,} fi fi nicName=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface external-ids:iface-id="$podName"."$namespace" | tr -d '\r') if [ -z "$nicName" ]; then echo "failed to find ovs interface for Pod namespacedPod on node $nodeName" exit 1 fi podNicType=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type}) podNetNs=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r' | sed -e 's/^"//' -e 's/"$//') if [ "$podNicType" != "internal-port" ]; then interface=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-vsctl --format=csv --data=bare --no-heading --columns=name find interface external_id:iface-id="$podName"."$namespace") peer=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ip link show $interface | grep -oE "^[0-9]+:\\s$interface@if[0-9]+" | awk -F @ '{print $2}') peerIndex=${peer//if/} peer=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ip link show type veth | grep "^$peerIndex:" | awk -F @ '{print $1}') nicName=$(echo $peer | awk '{print $2}') fi set +o pipefail master=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ip link show $nicName | grep -Eo '\smaster\s\w+\s' | awk '{print $2}') set -o pipefail if [ ! -z "$master" ]; then echo "Error: Pod nic $nicName is a slave of $master, please set the destination mac address." exit 1 fi if [[ "$gateway" =~ .*:.* ]]; then cmd="ndisc6 -q $gateway $nicName" output=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ndisc6 -q "$gateway" "$nicName") else cmd="arping -c3 -C1 -i1 -I $nicName $gateway" output=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" arping -c3 -C1 -i1 -I "$nicName" "$gateway") fi if [ $? -ne 0 ]; then echo "Error: failed to execute '$cmd' in Pod's netns" exit 1 fi dstMac=$(echo "$output" | grep -oE '([[:xdigit:]]{1,2}:){5}[[:xdigit:]]{1,2}') fi fi if [ -z "$dstMac" ]; then echo "Using the gateway mac address as destination" lr=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_router}) if [ -z "$lr" ]; then lr=$(kubectl get subnet "$ls" -o jsonpath={.spec.vpc}) fi dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=mac find logical_router_port name="$lr"-"$ls" | tr -d '\r') fi if [ -z "$dstMac" ]; then echo "failed to get destination mac" exit 1 fi lsp="$podName.$namespace" lspUUID=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=_uuid find logical_switch_port name="$lsp") if [ -z "$lspUUID" ]; then echo "Notice: LSP $lsp does not exist" fi vmOwner=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath='{.metadata.ownerReferences[?(@.kind=="VirtualMachineInstance")].name}') if [ ! -z "$vmOwner" ]; then lsp="$vmOwner.$namespace" fi if [ -z "$lsp" ]; then echo "failed to get LSP of Pod $namespace/$podName" exit 1 fi type="$3" case $type in icmp) set -x kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$lsp\" && ip.ttl == 64 && icmp && eth.src == $mac && ip$af.src == $podIP && eth.dst == $dstMac && ip$af.dst == $dst" ;; tcp|udp) set -x kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$lsp\" && ip.ttl == 64 && eth.src == $mac && ip$af.src == $podIP && eth.dst == $dstMac && ip$af.dst == $dst && $type.src == 10000 && $type.dst == $4" ;; *) echo "type $type not supported" echo "kubectl ko trace {namespace/podname} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp or udp port]" exit 1 ;; esac set +x echo "--------" echo "Start OVS Tracing" echo "" echo "" inPort=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-vsctl --format=csv --data=bare --no-heading --columns=ofport find interface external_id:iface-id="$podName"."$namespace") case $type in icmp) set -x kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,icmp$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$dstMac" ;; tcp|udp) set -x kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,$type$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$dstMac,${type}_src=1000,${type}_dst=$4" ;; *) echo "type $type not supported" echo "kubectl ko trace {namespace/podname} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp or udp port]" exit 1 ;; esac } xxctl(){ subcommand="$1"; shift nodeName="$1"; shift kubectl get no "$nodeName" > /dev/null ovsPod=$(kubectl get pod -n $KUBE_OVN_NS -l app=ovs -o 'jsonpath={.items[?(@.spec.nodeName=="'$nodeName'")].metadata.name}') if [ -z "$ovsPod" ]; then echo "ovs pod doesn't exist on node $nodeName" exit 1 fi kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-$subcommand "$@" } checkLeader(){ component="$1"; shift set +o pipefail count=$(kubectl get ep ovn-$component -n $KUBE_OVN_NS -o yaml | grep ip | wc -l) set -o pipefail if [ $count -eq 0 ]; then echo "no ovn-$component exists !!" exit 1 fi if [ $count -gt 1 ]; then echo "ovn-$component has more than one leader !!" exit 1 fi echo "ovn-$component leader check ok" } diagnose(){ kubectl get crd vpcs.kubeovn.io kubectl get crd vpc-nat-gateways.kubeovn.io kubectl get crd subnets.kubeovn.io kubectl get crd ips.kubeovn.io kubectl get crd vlans.kubeovn.io kubectl get crd provider-networks.kubeovn.io set +eu if ! kubectl get svc kube-dns -n kube-system ; then echo "Warning: kube-dns doesn't exist, maybe there is coredns service." fi set -eu kubectl get svc kubernetes -n default kubectl get sa -n kube-system ovn kubectl get clusterrole system:ovn kubectl get clusterrolebinding ovn kubectl get no -o wide kubectl ko nbctl show kubectl ko nbctl lr-policy-list ovn-cluster kubectl ko nbctl lr-route-list ovn-cluster kubectl ko nbctl ls-lb-list ovn-default kubectl ko nbctl list address_set kubectl ko nbctl list acl kubectl ko sbctl show if [ "${WITHOUT_KUBE_PROXY}" = "false" ]; then checkKubeProxy fi checkDeployment ovn-central checkDeployment kube-ovn-controller checkDaemonSet kube-ovn-cni checkDaemonSet ovs-ovn checkDeployment coredns checkLeader nb checkLeader sb checkLeader northd type="$1" case $type in all) echo "### kube-ovn-controller recent log" set +e kubectl logs -n $KUBE_OVN_NS -l app=kube-ovn-controller --tail=100 | grep E$(date +%m%d) set -e echo "" pingers=$(kubectl -n $KUBE_OVN_NS get po --no-headers -o custom-columns=NAME:.metadata.name -l app=kube-ovn-pinger) for pinger in $pingers do nodeName=$(kubectl get pod "$pinger" -n "$KUBE_OVN_NS" -o jsonpath={.spec.nodeName}) echo "### start to diagnose node $nodeName" echo "#### ovn-controller log:" kubectl exec -n $KUBE_OVN_NS "$pinger" -- tail /var/log/ovn/ovn-controller.log echo "" echo "#### ovs-vswitchd log:" kubectl exec -n $KUBE_OVN_NS "$pinger" -- tail /var/log/openvswitch/ovs-vswitchd.log echo "" echo "#### ovs-vsctl show results:" kubectl exec -n $KUBE_OVN_NS "$pinger" -- ovs-vsctl show echo "" echo "#### pinger diagnose results:" kubectl exec -n $KUBE_OVN_NS "$pinger" -- /kube-ovn/kube-ovn-pinger --mode=job echo "### finish diagnose node $nodeName" echo "" done ;; node) nodeName="$2" kubectl get no "$nodeName" > /dev/null pinger=$(kubectl -n $KUBE_OVN_NS get po -l app=kube-ovn-pinger -o 'jsonpath={.items[?(@.spec.nodeName=="'$nodeName'")].metadata.name}') if [ ! -n "$pinger" ]; then echo "Error: No kube-ovn-pinger running on node $nodeName" exit 1 fi echo "### start to diagnose node $nodeName" echo "#### ovn-controller log:" kubectl exec -n $KUBE_OVN_NS "$pinger" -- tail /var/log/ovn/ovn-controller.log echo "" echo "#### ovs-vswitchd log:" kubectl exec -n $KUBE_OVN_NS "$pinger" -- tail /var/log/openvswitch/ovs-vswitchd.log echo "" kubectl exec -n $KUBE_OVN_NS "$pinger" -- /kube-ovn/kube-ovn-pinger --mode=job echo "### finish diagnose node $nodeName" echo "" ;; *) echo "type $type not supported" echo "kubectl ko diagnose {all|node} [nodename]" ;; esac } getOvnCentralPod(){ NB_POD=$(kubectl get pod -n $KUBE_OVN_NS -l ovn-nb-leader=true | grep ovn-central | head -n 1 | awk '{print $1}') if [ -z "$NB_POD" ]; then echo "nb leader not exists" exit 1 fi OVN_NB_POD=$NB_POD SB_POD=$(kubectl get pod -n $KUBE_OVN_NS -l ovn-sb-leader=true | grep ovn-central | head -n 1 | awk '{print $1}') if [ -z "$SB_POD" ]; then echo "nb leader not exists" exit 1 fi OVN_SB_POD=$SB_POD VERSION=$(kubectl -n kube-system get pods -l ovn-sb-leader=true -o yaml | grep "image: $REGISTRY/kube-ovn:" | head -n 1 | awk -F ':' '{print $3}') if [ -z "$VERSION" ]; then echo "kubeovn version not exists" exit 1 fi KUBE_OVN_VERSION=$VERSION } checkDaemonSet(){ name="$1" currentScheduled=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.currentNumberScheduled}) desiredScheduled=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.desiredNumberScheduled}) available=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.numberAvailable}) ready=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.numberReady}) if [ "$currentScheduled" = "$desiredScheduled" ] && [ "$desiredScheduled" = "$available" ] && [ "$available" = "$ready" ]; then echo "ds $name ready" else echo "Error ds $name not ready" exit 1 fi } checkDeployment(){ name="$1" ready=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.readyReplicas}) updated=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.updatedReplicas}) desire=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.replicas}) available=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.availableReplicas}) if [ "$ready" = "$updated" ] && [ "$updated" = "$desire" ] && [ "$desire" = "$available" ]; then echo "deployment $name ready" else echo "Error deployment $name not ready" exit 1 fi } checkKubeProxy(){ if kubectl get ds -n kube-system --no-headers -o custom-columns=NAME:.metadata.name | grep '^kube-proxy$' >/dev/null; then checkDaemonSet kube-proxy else for node in $(kubectl get node --no-headers -o custom-columns=NAME:.metadata.name); do local pod=$(kubectl get pod -n $KUBE_OVN_NS -l app=kube-ovn-cni -o 'jsonpath={.items[?(@.spec.nodeName=="'$node'")].metadata.name}') local ip=$(kubectl get pod -n $KUBE_OVN_NS -l app=kube-ovn-cni -o 'jsonpath={.items[?(@.spec.nodeName=="'$node'")].status.podIP}') local arg="" if [[ $ip =~ .*:.* ]]; then arg="g6" ip="[$ip]" fi healthResult=$(kubectl -n $KUBE_OVN_NS exec $pod -- curl -s${arg} -m 3 -w %{http_code} http://$ip:10256/healthz -o /dev/null | grep -v 200 || true) if [ -n "$healthResult" ]; then echo "$node kube-proxy's health check failed" exit 1 fi done fi echo "kube-proxy ready" } dbtool(){ suffix=$(date +%m%d%H%M%s) component="$1"; shift action="$1"; shift case $component in nb) case $action in status) kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl ovsdb-server/get-db-storage-status OVN_Northbound ;; kick) kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/kick OVN_Northbound "$1" ;; backup) kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnnb_db.$suffix.backup /etc/ovn/ovnnb_db.db kubectl cp $KUBE_OVN_NS/$OVN_NB_POD:/etc/ovn/ovnnb_db.$suffix.backup $(pwd)/ovnnb_db.$suffix.backup kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /etc/ovn/ovnnb_db.$suffix.backup echo "backup ovn-$component db to $(pwd)/ovnnb_db.$suffix.backup" ;; dbstatus) kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-appctl -t /var/run/ovn/ovnnb_db.ctl ovsdb-server/get-db-storage-status OVN_Northbound ;; restore) # set ovn-central replicas to 0 replicas=$(kubectl get deployment -n $KUBE_OVN_NS ovn-central -o jsonpath={.spec.replicas}) kubectl scale deployment -n $KUBE_OVN_NS ovn-central --replicas=0 echo "ovn-central original replicas is $replicas" # backup ovn-nb db declare nodeIpArray declare podNameArray declare nodeIps if [[ $(kubectl get deployment -n kube-system ovn-central -o jsonpath='{.spec.template.spec.containers[0].env[1]}') =~ "NODE_IPS" ]]; then nodeIpVals=$(kubectl get deployment -n kube-system ovn-central -o jsonpath='{.spec.template.spec.containers[0].env[1].value}') nodeIps=(${nodeIpVals//,/ }) else nodeIps=$(kubectl get node -lkube-ovn/role=master -o wide | grep -v "INTERNAL-IP" | awk '{print $6}') fi firstIP=${nodeIps[0]} podNames=$(kubectl get pod -n $KUBE_OVN_NS | grep ovs-ovn | awk '{print $1}') echo "first nodeIP is $firstIP" i=0 for nodeIp in ${nodeIps[@]} do for pod in $podNames do hostip=$(kubectl get pod -n $KUBE_OVN_NS $pod -o jsonpath={.status.hostIP}) if [ $nodeIp = $hostip ]; then nodeIpArray[$i]=$nodeIp podNameArray[$i]=$pod i=$(expr $i + 1) echo "ovs-ovn pod on node $nodeIp is $pod" break fi done done echo "backup nb db file" kubectl exec -it -n $KUBE_OVN_NS ${podNameArray[0]} -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnnb_db_standalone.db /etc/ovn/ovnnb_db.db # mv all db files for pod in ${podNameArray[@]} do kubectl exec -it -n $KUBE_OVN_NS $pod -- mv /etc/ovn/ovnnb_db.db /tmp kubectl exec -it -n $KUBE_OVN_NS $pod -- mv /etc/ovn/ovnsb_db.db /tmp done # restore db and replicas echo "restore nb db file, operate in pod ${podNameArray[0]}" kubectl exec -it -n $KUBE_OVN_NS ${podNameArray[0]} -- mv /etc/ovn/ovnnb_db_standalone.db /etc/ovn/ovnnb_db.db kubectl scale deployment -n $KUBE_OVN_NS ovn-central --replicas=$replicas echo "finish restore nb db file and ovn-central replicas" echo "recreate ovs-ovn pods" kubectl delete pod -n $KUBE_OVN_NS -l app=ovs ;; *) echo "unknown action $action" esac ;; sb) case $action in status) kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl ovsdb-server/get-db-storage-status OVN_Southbound ;; kick) kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/kick OVN_Southbound "$1" ;; backup) kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnsb_db.$suffix.backup /etc/ovn/ovnsb_db.db kubectl cp $KUBE_OVN_NS/$OVN_SB_POD:/etc/ovn/ovnsb_db.$suffix.backup $(pwd)/ovnsb_db.$suffix.backup kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /etc/ovn/ovnsb_db.$suffix.backup echo "backup ovn-$component db to $(pwd)/ovnsb_db.$suffix.backup" ;; dbstatus) kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-appctl -t /var/run/ovn/ovnsb_db.ctl ovsdb-server/get-db-storage-status OVN_Southbound ;; restore) echo "restore cmd is only used for nb db" ;; *) echo "unknown action $action" esac ;; *) echo "unknown subcommand $component" esac } tuning(){ action="$1"; shift sys="$1"; shift case $action in install-fastpath) case $sys in centos7) docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp/:/tmp/ $REGISTRY/centos7-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh centos install" while [ ! -f /tmp/kube_ovn_fastpath.ko ]; do sleep 1 done for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); do kubectl cp /tmp/kube_ovn_fastpath.ko kube-system/"$i":/tmp/ done ;; centos8) docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp/:/tmp/ $REGISTRY/centos8-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh centos install" while [ ! -f /tmp/kube_ovn_fastpath.ko ]; do sleep 1 done for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); do kubectl cp /tmp/kube_ovn_fastpath.ko kube-system/"$i":/tmp/ done ;; *) echo "unknown system $sys" esac ;; local-install-fastpath) case $sys in centos7) # shellcheck disable=SC2145 docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos7-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh centos local-install $@" for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); do kubectl cp /tmp/kube_ovn_fastpath.ko kube-system/"$i":/tmp/ done ;; centos8) # shellcheck disable=SC2145 docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos8-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh centos local-install $@" for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); do kubectl cp /tmp/kube_ovn_fastpath.ko kube-system/"$i":/tmp/ done ;; *) echo "unknown system $sys" esac ;; remove-fastpath) case $sys in centos) for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); do kubectl -n kube-system exec "$i" -- rm -f /tmp/kube_ovn_fastpath.ko done ;; *) echo "unknown system $sys" esac ;; install-stt) case $sys in centos7) # shellcheck disable=SC2145 docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos7-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh stt install" for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); do for k in /tmp/*.rpm; do kubectl cp "$k" kube-system/"$i":/tmp/ done done ;; centos8) # shellcheck disable=SC2145 docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos8-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh stt install" for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); do for k in /tmp/*.rpm; do kubectl cp "$k" kube-system/"$i":/tmp/ done done ;; *) echo "unknown system $sys" esac ;; local-install-stt) case $sys in centos7) # shellcheck disable=SC2145 docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos7-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh stt local-install $@" for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); do for k in /tmp/*.rpm; do kubectl cp "$k" kube-system/"$i":/tmp/ done done ;; centos8) # shellcheck disable=SC2145 docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos8-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh stt local-install $@" for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); do for k in /tmp/*.rpm; do kubectl cp "$k" kube-system/"$i":/tmp/ done done ;; *) echo "unknown system $sys" esac ;; remove-stt) case $sys in centos) for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); do kubectl -n kube-system exec "$i" -- rm -f /tmp/openvswitch-kmod*.rpm done ;; *) echo "unknown system $sys" esac ;; *) echo "unknown action $action" esac } reload(){ kubectl delete pod -n kube-system -l app=ovn-central kubectl rollout status deployment/ovn-central -n kube-system kubectl delete pod -n kube-system -l app=ovs kubectl delete pod -n kube-system -l app=kube-ovn-controller kubectl rollout status deployment/kube-ovn-controller -n kube-system kubectl delete pod -n kube-system -l app=kube-ovn-cni kubectl rollout status daemonset/kube-ovn-cni -n kube-system kubectl delete pod -n kube-system -l app=kube-ovn-pinger kubectl rollout status daemonset/kube-ovn-pinger -n kube-system kubectl delete pod -n kube-system -l app=kube-ovn-monitor kubectl rollout status deployment/kube-ovn-monitor -n kube-system } env-check(){ set +e KUBE_OVN_NS=kube-system podNames=$(kubectl get pod --no-headers -n $KUBE_OVN_NS | grep kube-ovn-cni | awk '{print $1}') for pod in $podNames do nodeName=$(kubectl get pod $pod -n $KUBE_OVN_NS -o jsonpath={.spec.nodeName}) echo "************************************************" echo "Start environment check for Node $nodeName" echo "************************************************" kubectl exec -it -n $KUBE_OVN_NS $pod -c cni-server -- bash /kube-ovn/env-check.sh done } if [ $# -lt 1 ]; then showHelp exit 0 else subcommand="$1"; shift fi getOvnCentralPod case $subcommand in nbctl) kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl "$@" ;; sbctl) kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-sbctl "$@" ;; vsctl|ofctl|dpctl|appctl) xxctl "$subcommand" "$@" ;; nb|sb) dbtool "$subcommand" "$@" ;; tcpdump) tcpdump "$@" ;; trace) trace "$@" ;; diagnose) diagnose "$@" ;; reload) reload ;; tuning) tuning "$@" ;; env-check) env-check ;; *) showHelp exit 1 ;; esac `)))
View Source
var Multus = template.Must(template.New("multus-network-plugin.yaml").Parse( dedent.Dedent(` --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: network-attachment-definitions.k8s.cni.cncf.io spec: group: k8s.cni.cncf.io scope: Namespaced names: plural: network-attachment-definitions singular: network-attachment-definition kind: NetworkAttachmentDefinition shortNames: - net-attach-def versions: - name: v1 served: true storage: true schema: openAPIV3Schema: description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing Working Group to express the intent for attaching pods to one or more logical or physical networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec' type: object properties: apiVersion: description: 'APIVersion defines the versioned schema of this represen tation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment' type: object properties: config: description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration' type: string --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: multus rules: - apiGroups: ["k8s.cni.cncf.io"] resources: - '*' verbs: - '*' - apiGroups: - "" resources: - pods - pods/status verbs: - get - update - apiGroups: - "" - events.k8s.io resources: - events verbs: - create - patch - update --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: multus roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: multus subjects: - kind: ServiceAccount name: multus namespace: kube-system --- apiVersion: v1 kind: ServiceAccount metadata: name: multus namespace: kube-system --- kind: ConfigMap apiVersion: v1 metadata: name: multus-cni-config namespace: kube-system labels: tier: node app: multus data: # NOTE: If you'd prefer to manually apply a configuration file, you may create one here. # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod # change the "args" line below from # - "--multus-conf-file=auto" # to: # "--multus-conf-file=/tmp/multus-conf/70-multus.conf" # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet. cni-conf.json: | { "name": "multus-cni-network", "type": "multus", "capabilities": { "portMappings": true }, "delegates": [ { "cniVersion": "0.3.1", "name": "default-cni-network", "plugins": [ { "type": "flannel", "name": "flannel.1", "delegate": { "isDefaultGateway": true, "hairpinMode": true } }, { "type": "portmap", "capabilities": { "portMappings": true } } ] } ], "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig" } --- apiVersion: apps/v1 kind: DaemonSet metadata: name: kube-multus-ds namespace: kube-system labels: tier: node app: multus name: multus spec: selector: matchLabels: name: multus updateStrategy: type: RollingUpdate template: metadata: labels: tier: node app: multus name: multus spec: hostNetwork: true tolerations: - operator: Exists effect: NoSchedule serviceAccountName: multus containers: - name: kube-multus image: {{ .MultusImage }} command: ["/entrypoint.sh"] args: - "--multus-conf-file=auto" - "--cni-version=0.3.1" resources: requests: cpu: "100m" memory: "50Mi" limits: cpu: "100m" memory: "50Mi" securityContext: privileged: true volumeMounts: - name: cni mountPath: /host/etc/cni/net.d - name: cnibin mountPath: /host/opt/cni/bin - name: multus-cfg mountPath: /tmp/multus-conf terminationGracePeriodSeconds: 10 volumes: - name: cni hostPath: path: /etc/cni/net.d - name: cnibin hostPath: path: /opt/cni/bin - name: multus-cfg configMap: name: multus-cni-config items: - key: cni-conf.json path: 70-multus.conf `)))
Functions ¶
This section is empty.
Types ¶
This section is empty.
Source Files
Click to show internal directories.
Click to hide internal directories.