Rename "kube-peers" to "kube-utils"
It's now doing things like network status that are outside of the scope of "peers".
This commit is contained in:
1 parent
9b0d1bd054
commit
2a04df1049
7 files changed
+13
-13
No files matched your search
@@ -0,0 +1,210 @@
|
||||
/*
|
||||
In order to keep track of active weave peers, we use annotations on the Kubernetes cluster.
|
||||
|
||||
Kubernetes uses etcd to distribute and synchronise these annotations so we don't have to.
|
||||
|
||||
This module deals with operations on the peerlist backed by Kubernetes' annotation mechanism.
|
||||
*/
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
kubeErrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
api "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
wait "k8s.io/apimachinery/pkg/util/wait"
|
||||
kubernetes "k8s.io/client-go/kubernetes"
|
||||
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
|
||||
)
|
||||
|
||||
type configMapAnnotations struct {
|
||||
ConfigMapName string
|
||||
Namespace string
|
||||
Client corev1client.ConfigMapsGetter
|
||||
cm *v1.ConfigMap
|
||||
}
|
||||
|
||||
func newConfigMapAnnotations(ns string, configMapName string, clientset *kubernetes.Clientset) *configMapAnnotations {
|
||||
return &configMapAnnotations{
|
||||
Namespace: ns,
|
||||
ConfigMapName: configMapName,
|
||||
Client: clientset.CoreV1(),
|
||||
}
|
||||
}
|
||||
|
||||
type peerList struct {
|
||||
Peers []peerInfo
|
||||
}
|
||||
|
||||
type peerInfo struct {
|
||||
PeerName string // Weave internal unique ID
|
||||
NodeName string // Kubernetes node name
|
||||
}
|
||||
|
||||
func (pl peerList) contains(peerName string) bool {
|
||||
for _, peer := range pl.Peers {
|
||||
if peer.PeerName == peerName {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (pl *peerList) add(peerName string, name string) {
|
||||
pl.Peers = append(pl.Peers, peerInfo{PeerName: peerName, NodeName: name})
|
||||
}
|
||||
|
||||
func (pl *peerList) remove(peerNameToRemove string) {
|
||||
for i := 0; i < len(pl.Peers); {
|
||||
if pl.Peers[i].PeerName == peerNameToRemove {
|
||||
pl.Peers = append(pl.Peers[:i], pl.Peers[i+1:]...)
|
||||
} else {
|
||||
i++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
retryPeriod = time.Second * 2
|
||||
jitterFactor = 1.0
|
||||
|
||||
// Prefix all our annotation keys with this string so they don't clash with anyone else's
|
||||
KubePeersPrefix = "kube-peers.weave.works/"
|
||||
// KubePeersAnnotationKey is the default annotation key
|
||||
KubePeersAnnotationKey = KubePeersPrefix + "peers"
|
||||
)
|
||||
|
||||
func (cml *configMapAnnotations) Init() error {
|
||||
for {
|
||||
// Since it's potentially racy to GET, then CREATE if not found, we wrap in a check loop
|
||||
// so that if the configmap is created after our GET but before or CREATE, we'll gracefully
|
||||
// re-try to get the configmap.
|
||||
var err error
|
||||
cml.cm, err = cml.Client.ConfigMaps(cml.Namespace).Get(cml.ConfigMapName, api.GetOptions{})
|
||||
if err != nil {
|
||||
if !kubeErrors.IsNotFound(err) {
|
||||
return errors.Wrapf(err, "Unable to fetch ConfigMap %s/%s", cml.Namespace, cml.ConfigMapName)
|
||||
}
|
||||
cml.cm, err = cml.Client.ConfigMaps(cml.Namespace).Create(&v1.ConfigMap{
|
||||
ObjectMeta: api.ObjectMeta{
|
||||
Name: cml.ConfigMapName,
|
||||
Namespace: cml.Namespace,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
if kubeErrors.IsAlreadyExists(err) {
|
||||
continue
|
||||
}
|
||||
return errors.Wrapf(err, "Unable to create ConfigMap %s/%s", cml.Namespace, cml.ConfigMapName)
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
if cml.cm.Annotations == nil {
|
||||
cml.cm.Annotations = make(map[string]string)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cml *configMapAnnotations) GetPeerList() (*peerList, error) {
|
||||
var record peerList
|
||||
if cml.cm == nil {
|
||||
return nil, errors.New("endpoint not initialized, call Init first")
|
||||
}
|
||||
if recordBytes, found := cml.cm.Annotations[KubePeersAnnotationKey]; found {
|
||||
if err := json.Unmarshal([]byte(recordBytes), &record); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return &record, nil
|
||||
}
|
||||
|
||||
func (cml *configMapAnnotations) UpdatePeerList(list peerList) error {
|
||||
recordBytes, err := json.Marshal(list)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return cml.UpdateAnnotation(KubePeersAnnotationKey, string(recordBytes))
|
||||
}
|
||||
|
||||
// Clean up a string so it meets the Kubernetes requiremements for Annotation keys:
|
||||
// name part must consist of alphanumeric characters, '-', '_' or '.', and must
|
||||
// start and end with an alphanumeric character (e.g. 'MyName', or 'my.name', or '123-abc')
|
||||
func cleanKey(key string) string {
|
||||
buf := []byte(key)
|
||||
for i, c := range buf {
|
||||
if (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' || c == '.' || c == '/' {
|
||||
continue
|
||||
}
|
||||
buf[i] = '_'
|
||||
}
|
||||
return string(buf)
|
||||
}
|
||||
|
||||
func (cml *configMapAnnotations) GetAnnotation(key string) (string, bool) {
|
||||
value, ok := cml.cm.Annotations[cleanKey(key)]
|
||||
return value, ok
|
||||
}
|
||||
|
||||
func (cml *configMapAnnotations) UpdateAnnotation(key, value string) (err error) {
|
||||
if cml.cm == nil {
|
||||
return errors.New("endpoint not initialized, call Init first")
|
||||
}
|
||||
// speculatively change the state, then replace with whatever comes back
|
||||
// from Update(), which will be the latest on the server whatever happened
|
||||
cml.cm.Annotations[cleanKey(key)] = value
|
||||
cml.cm, err = cml.Client.ConfigMaps(cml.Namespace).Update(cml.cm)
|
||||
return err
|
||||
}
|
||||
|
||||
func (cml *configMapAnnotations) RemoveAnnotation(key string) (err error) {
|
||||
if cml.cm == nil {
|
||||
return errors.New("endpoint not initialized, call Init first")
|
||||
}
|
||||
// speculatively change the state, then replace with whatever comes back
|
||||
// from Update(), which will be the latest on the server whatever happened
|
||||
delete(cml.cm.Annotations, cleanKey(key))
|
||||
cml.cm, err = cml.Client.ConfigMaps(cml.Namespace).Update(cml.cm)
|
||||
return err
|
||||
}
|
||||
|
||||
func (cml *configMapAnnotations) RemoveAnnotationsWithValue(valueToRemove string) (err error) {
|
||||
if cml.cm == nil {
|
||||
return errors.New("endpoint not initialized, call Init first")
|
||||
}
|
||||
// speculatively change the state, then replace with whatever comes back
|
||||
// from Update(), which will be the latest on the server whatever happened
|
||||
for key, value := range cml.cm.Annotations {
|
||||
if value == valueToRemove {
|
||||
delete(cml.cm.Annotations, key) // don't need to clean this key as it came from the map
|
||||
}
|
||||
}
|
||||
cml.cm, err = cml.Client.ConfigMaps(cml.Namespace).Update(cml.cm)
|
||||
return err
|
||||
}
|
||||
|
||||
// Loop with jitter, fetching the cml data and calling f() until it
|
||||
// doesn't get an optimistic locking conflict.
|
||||
// If it succeeds or gets any other kind of error, stop the loop.
|
||||
func (cml *configMapAnnotations) LoopUpdate(f func() error) error {
|
||||
stop := make(chan struct{})
|
||||
var err error
|
||||
wait.JitterUntil(func() {
|
||||
if err = cml.Init(); err != nil {
|
||||
close(stop)
|
||||
return
|
||||
}
|
||||
err = f()
|
||||
if err != nil && kubeErrors.IsConflict(err) {
|
||||
log.Printf("Optimistic locking conflict: trying again: %s", err)
|
||||
return
|
||||
}
|
||||
close(stop)
|
||||
}, retryPeriod, jitterFactor, true, stop)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
/*
|
||||
Package main deals with weave-net peers on the cluster.
|
||||
|
||||
This involves peer management, such as getting the latest peers or removing defunct peers from the cluster
|
||||
*/
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
|
||||
api "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
|
||||
weaveapi "github.com/weaveworks/weave/api"
|
||||
"github.com/weaveworks/weave/common"
|
||||
)
|
||||
|
||||
type nodeInfo struct {
|
||||
name string
|
||||
addr string
|
||||
}
|
||||
|
||||
// return the IP addresses of all nodes in the cluster
|
||||
func getKubePeers(c *kubernetes.Clientset) ([]nodeInfo, error) {
|
||||
nodeList, err := c.CoreV1().Nodes().List(api.ListOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
addresses := make([]nodeInfo, 0, len(nodeList.Items))
|
||||
for _, peer := range nodeList.Items {
|
||||
var internalIP, externalIP string
|
||||
for _, addr := range peer.Status.Addresses {
|
||||
// Check it's a valid ipv4 address
|
||||
ip := net.ParseIP(addr.Address)
|
||||
if ip == nil || ip.To4() == nil {
|
||||
continue
|
||||
}
|
||||
if addr.Type == "InternalIP" {
|
||||
internalIP = ip.To4().String()
|
||||
} else if addr.Type == "ExternalIP" {
|
||||
externalIP = ip.To4().String()
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback for cases where a Node has an ExternalIP but no InternalIP
|
||||
if internalIP != "" {
|
||||
addresses = append(addresses, nodeInfo{name: peer.Name, addr: internalIP})
|
||||
} else if externalIP != "" {
|
||||
addresses = append(addresses, nodeInfo{name: peer.Name, addr: externalIP})
|
||||
}
|
||||
}
|
||||
return addresses, nil
|
||||
}
|
||||
|
||||
const (
|
||||
configMapName = "weave-net"
|
||||
configMapNamespace = "kube-system"
|
||||
)
|
||||
|
||||
// update the list of all peers that have gone through this code path
|
||||
func addMyselfToPeerList(cml *configMapAnnotations, c *kubernetes.Clientset, peerName, name string) (*peerList, error) {
|
||||
var list *peerList
|
||||
err := cml.LoopUpdate(func() error {
|
||||
var err error
|
||||
list, err = cml.GetPeerList()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !list.contains(peerName) {
|
||||
list.add(peerName, name)
|
||||
err = cml.UpdatePeerList(*list)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return list, err
|
||||
}
|
||||
|
||||
func checkIamInPeerList(cml *configMapAnnotations, c *kubernetes.Clientset, peerName string) (bool, error) {
|
||||
if err := cml.Init(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
list, err := cml.GetPeerList()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
common.Log.Debugf("[kube-peers] Checking peer %q against list %v", peerName, list)
|
||||
return list.contains(peerName), nil
|
||||
}
|
||||
|
||||
// For each of those peers that is no longer listed as a node by
|
||||
// Kubernetes, remove it from Weave IPAM
|
||||
func reclaimRemovedPeers(weave *weaveapi.Client, cml *configMapAnnotations, nodes []nodeInfo, myPeerName string) error {
|
||||
for {
|
||||
if err := cml.Init(); err != nil {
|
||||
return err
|
||||
}
|
||||
// 1. Compare peers stored in the peerList against all peers reported by k8s now.
|
||||
storedPeerList, err := cml.GetPeerList()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
peerMap := make(map[string]peerInfo, len(storedPeerList.Peers))
|
||||
for _, peer := range storedPeerList.Peers {
|
||||
peerMap[peer.NodeName] = peer
|
||||
}
|
||||
for _, node := range nodes {
|
||||
delete(peerMap, node.name)
|
||||
}
|
||||
common.Log.Debugln("[kube-peers] Nodes that have disappeared:", peerMap)
|
||||
if len(peerMap) == 0 {
|
||||
break
|
||||
}
|
||||
// 2. Loop for each X in the first set and not in the second - we wish to remove X from our data structures
|
||||
for _, peer := range peerMap {
|
||||
common.Log.Debugln("[kube-peers] Preparing to remove disappeared peer", peer)
|
||||
okToRemove := false
|
||||
// 3. Check if there is an existing annotation with key X
|
||||
if existingAnnotation, found := cml.GetAnnotation(KubePeersPrefix + peer.PeerName); found {
|
||||
common.Log.Debugln("[kube-peers] Existing annotation", existingAnnotation)
|
||||
// 4. If annotation already contains my identity, ok;
|
||||
if existingAnnotation == myPeerName {
|
||||
okToRemove = true
|
||||
}
|
||||
} else {
|
||||
// 5. If non-existent, write an annotation with key X and contents "my identity"
|
||||
common.Log.Debugln("[kube-peers] Noting I plan to remove ", peer.PeerName)
|
||||
if err := cml.UpdateAnnotation(KubePeersPrefix+peer.PeerName, myPeerName); err == nil {
|
||||
okToRemove = true
|
||||
} else {
|
||||
common.Log.Debugln("[kube-peers] error from UpdateAnnotation: ", err)
|
||||
}
|
||||
}
|
||||
if okToRemove {
|
||||
// 6. If step 4 or 5 succeeded, rmpeer X
|
||||
result, err := weave.RmPeer(peer.PeerName)
|
||||
common.Log.Infof("[kube-peers] rmpeer of %s: %s", peer.PeerName, result)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cml.LoopUpdate(func() error {
|
||||
// 7aa. Remove any annotations Z* that have contents X
|
||||
if err := cml.RemoveAnnotationsWithValue(peer.PeerName); err != nil {
|
||||
return err
|
||||
}
|
||||
// 7a. Remove X from peerList
|
||||
storedPeerList.remove(peer.PeerName)
|
||||
if err := cml.UpdatePeerList(*storedPeerList); err != nil {
|
||||
return err
|
||||
}
|
||||
// 7b. Remove annotation with key X
|
||||
return cml.RemoveAnnotation(KubePeersPrefix + peer.PeerName)
|
||||
})
|
||||
}
|
||||
// 8. If step 5 failed due to optimistic lock conflict, stop: someone else is handling X
|
||||
|
||||
// Step 3-5 is to protect against two simultaneous rmpeers of X
|
||||
// Step 4 is to pick up again after a restart between step 5 and step 7b
|
||||
// If the peer doing the reclaim disappears between steps 5 and 7a, then someone will clean it up in step 7aa
|
||||
// If peer doing the reclaim disappears forever between 7a and 7b then we get a dangling annotation
|
||||
// This should be sufficiently rare that we don't care.
|
||||
}
|
||||
|
||||
// 9. Go back to step 1 until there is no difference between the two sets
|
||||
}
|
||||
// Question: Should we narrow step 2 by checking against Weave Net IPAM?
|
||||
// i.e. If peer X owns any address space and is marked unreachable, we want to rmpeer X
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
var (
|
||||
justReclaim bool
|
||||
justCheck bool
|
||||
justSetNodeStatus bool
|
||||
peerName string
|
||||
nodeName string
|
||||
logLevel string
|
||||
)
|
||||
flag.BoolVar(&justReclaim, "reclaim", false, "reclaim IP space from dead peers")
|
||||
flag.BoolVar(&justCheck, "check-peer-new", false, "return success if peer name is not stored in annotation")
|
||||
flag.BoolVar(&justSetNodeStatus, "set-node-status", false, "set NodeNetworkUnavailable to false")
|
||||
flag.StringVar(&peerName, "peer-name", "unknown", "name of this Weave Net peer")
|
||||
flag.StringVar(&nodeName, "node-name", "unknown", "name of this Kubernetes node")
|
||||
flag.StringVar(&logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
|
||||
flag.Parse()
|
||||
|
||||
common.SetLogLevel(logLevel)
|
||||
config, err := rest.InClusterConfig()
|
||||
if err != nil {
|
||||
common.Log.Fatalf("[kube-peers] Could not get cluster config: %v", err)
|
||||
}
|
||||
c, err := kubernetes.NewForConfig(config)
|
||||
if err != nil {
|
||||
common.Log.Fatalf("[kube-peers] Could not make Kubernetes connection: %v", err)
|
||||
}
|
||||
if justCheck {
|
||||
cml := newConfigMapAnnotations(configMapNamespace, configMapName, c)
|
||||
exists, err := checkIamInPeerList(cml, c, peerName)
|
||||
if err != nil {
|
||||
common.Log.Fatalf("[kube-peers] Could not check peer list: %v", err)
|
||||
}
|
||||
if exists {
|
||||
os.Exit(9)
|
||||
} else {
|
||||
os.Exit(0)
|
||||
}
|
||||
}
|
||||
if justSetNodeStatus {
|
||||
err := setNodeNetworkUnavailableFalse(c, nodeName)
|
||||
if err != nil {
|
||||
common.Log.Fatalf("[kube-peers] could not set node status: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
peers, err := getKubePeers(c)
|
||||
if err != nil {
|
||||
common.Log.Fatalf("[kube-peers] Could not get peers: %v", err)
|
||||
}
|
||||
if justReclaim {
|
||||
cml := newConfigMapAnnotations(configMapNamespace, configMapName, c)
|
||||
|
||||
list, err := addMyselfToPeerList(cml, c, peerName, nodeName)
|
||||
if err != nil {
|
||||
common.Log.Fatalf("[kube-peers] Could not update peer list: %v", err)
|
||||
}
|
||||
common.Log.Infoln("[kube-peers] Added myself to peer list", list)
|
||||
|
||||
weave := weaveapi.NewClient(os.Getenv("WEAVE_HTTP_ADDR"), common.Log)
|
||||
err = reclaimRemovedPeers(weave, cml, peers, peerName)
|
||||
if err != nil {
|
||||
common.Log.Fatalf("[kube-peers] Error while reclaiming space: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
for _, node := range peers {
|
||||
fmt.Println(node.addr)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
// based on k8s.io/kubernetes/pkg/controller/nodeipam/ipam/adapter.go
|
||||
func setNodeNetworkUnavailableFalse(c *kubernetes.Clientset, nodeName string) error {
|
||||
condition := v1.NodeCondition{
|
||||
Type: v1.NodeNetworkUnavailable,
|
||||
Status: v1.ConditionFalse,
|
||||
Reason: "WeaveIsUp",
|
||||
Message: "Weave pod has set this",
|
||||
LastTransitionTime: metav1.Now(),
|
||||
LastHeartbeatTime: metav1.Now(),
|
||||
}
|
||||
raw, err := json.Marshal(&[]v1.NodeCondition{condition})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
patch := []byte(fmt.Sprintf(`{"status":{"conditions":%s}}`, raw))
|
||||
_, err = c.CoreV1().Nodes().PatchStatus(nodeName, patch)
|
||||
return err
|
||||
}
|
||||
Reference in new issue
Block a user