Rename "kube-peers" to "kube-utils"

It's now doing things like network status that are outside of the
scope of "peers".
This commit is contained in:
Bryan Boreham committed 2018-06-11 11:22:07 +00:00
1 parent 9b0d1bd054
commit 2a04df1049
7 files changed
+13 -13

No files matched your search

+210
View File
@@ -0,0 +1,210 @@
/*
In order to keep track of active weave peers, we use annotations on the Kubernetes cluster.
Kubernetes uses etcd to distribute and synchronise these annotations so we don't have to.
This module deals with operations on the peerlist backed by Kubernetes' annotation mechanism.
*/
package main
import (
"encoding/json"
"log"
"time"
"github.com/pkg/errors"
v1 "k8s.io/api/core/v1"
kubeErrors "k8s.io/apimachinery/pkg/api/errors"
api "k8s.io/apimachinery/pkg/apis/meta/v1"
wait "k8s.io/apimachinery/pkg/util/wait"
kubernetes "k8s.io/client-go/kubernetes"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
)
type configMapAnnotations struct {
ConfigMapName string
Namespace string
Client corev1client.ConfigMapsGetter
cm *v1.ConfigMap
}
func newConfigMapAnnotations(ns string, configMapName string, clientset *kubernetes.Clientset) *configMapAnnotations {
return &configMapAnnotations{
Namespace: ns,
ConfigMapName: configMapName,
Client: clientset.CoreV1(),
}
}
type peerList struct {
Peers []peerInfo
}
type peerInfo struct {
PeerName string // Weave internal unique ID
NodeName string // Kubernetes node name
}
func (pl peerList) contains(peerName string) bool {
for _, peer := range pl.Peers {
if peer.PeerName == peerName {
return true
}
}
return false
}
func (pl *peerList) add(peerName string, name string) {
pl.Peers = append(pl.Peers, peerInfo{PeerName: peerName, NodeName: name})
}
func (pl *peerList) remove(peerNameToRemove string) {
for i := 0; i < len(pl.Peers); {
if pl.Peers[i].PeerName == peerNameToRemove {
pl.Peers = append(pl.Peers[:i], pl.Peers[i+1:]...)
} else {
i++
}
}
}
const (
retryPeriod = time.Second * 2
jitterFactor = 1.0
// Prefix all our annotation keys with this string so they don't clash with anyone else's
KubePeersPrefix = "kube-peers.weave.works/"
// KubePeersAnnotationKey is the default annotation key
KubePeersAnnotationKey = KubePeersPrefix + "peers"
)
func (cml *configMapAnnotations) Init() error {
for {
// Since it's potentially racy to GET, then CREATE if not found, we wrap in a check loop
// so that if the configmap is created after our GET but before or CREATE, we'll gracefully
// re-try to get the configmap.
var err error
cml.cm, err = cml.Client.ConfigMaps(cml.Namespace).Get(cml.ConfigMapName, api.GetOptions{})
if err != nil {
if !kubeErrors.IsNotFound(err) {
return errors.Wrapf(err, "Unable to fetch ConfigMap %s/%s", cml.Namespace, cml.ConfigMapName)
}
cml.cm, err = cml.Client.ConfigMaps(cml.Namespace).Create(&v1.ConfigMap{
ObjectMeta: api.ObjectMeta{
Name: cml.ConfigMapName,
Namespace: cml.Namespace,
},
})
if err != nil {
if kubeErrors.IsAlreadyExists(err) {
continue
}
return errors.Wrapf(err, "Unable to create ConfigMap %s/%s", cml.Namespace, cml.ConfigMapName)
}
}
break
}
if cml.cm.Annotations == nil {
cml.cm.Annotations = make(map[string]string)
}
return nil
}
func (cml *configMapAnnotations) GetPeerList() (*peerList, error) {
var record peerList
if cml.cm == nil {
return nil, errors.New("endpoint not initialized, call Init first")
}
if recordBytes, found := cml.cm.Annotations[KubePeersAnnotationKey]; found {
if err := json.Unmarshal([]byte(recordBytes), &record); err != nil {
return nil, err
}
}
return &record, nil
}
func (cml *configMapAnnotations) UpdatePeerList(list peerList) error {
recordBytes, err := json.Marshal(list)
if err != nil {
return err
}
return cml.UpdateAnnotation(KubePeersAnnotationKey, string(recordBytes))
}
// Clean up a string so it meets the Kubernetes requiremements for Annotation keys:
// name part must consist of alphanumeric characters, '-', '_' or '.', and must
// start and end with an alphanumeric character (e.g. 'MyName', or 'my.name', or '123-abc')
func cleanKey(key string) string {
buf := []byte(key)
for i, c := range buf {
if (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' || c == '.' || c == '/' {
continue
}
buf[i] = '_'
}
return string(buf)
}
func (cml *configMapAnnotations) GetAnnotation(key string) (string, bool) {
value, ok := cml.cm.Annotations[cleanKey(key)]
return value, ok
}
func (cml *configMapAnnotations) UpdateAnnotation(key, value string) (err error) {
if cml.cm == nil {
return errors.New("endpoint not initialized, call Init first")
}
// speculatively change the state, then replace with whatever comes back
// from Update(), which will be the latest on the server whatever happened
cml.cm.Annotations[cleanKey(key)] = value
cml.cm, err = cml.Client.ConfigMaps(cml.Namespace).Update(cml.cm)
return err
}
func (cml *configMapAnnotations) RemoveAnnotation(key string) (err error) {
if cml.cm == nil {
return errors.New("endpoint not initialized, call Init first")
}
// speculatively change the state, then replace with whatever comes back
// from Update(), which will be the latest on the server whatever happened
delete(cml.cm.Annotations, cleanKey(key))
cml.cm, err = cml.Client.ConfigMaps(cml.Namespace).Update(cml.cm)
return err
}
func (cml *configMapAnnotations) RemoveAnnotationsWithValue(valueToRemove string) (err error) {
if cml.cm == nil {
return errors.New("endpoint not initialized, call Init first")
}
// speculatively change the state, then replace with whatever comes back
// from Update(), which will be the latest on the server whatever happened
for key, value := range cml.cm.Annotations {
if value == valueToRemove {
delete(cml.cm.Annotations, key) // don't need to clean this key as it came from the map
}
}
cml.cm, err = cml.Client.ConfigMaps(cml.Namespace).Update(cml.cm)
return err
}
// Loop with jitter, fetching the cml data and calling f() until it
// doesn't get an optimistic locking conflict.
// If it succeeds or gets any other kind of error, stop the loop.
func (cml *configMapAnnotations) LoopUpdate(f func() error) error {
stop := make(chan struct{})
var err error
wait.JitterUntil(func() {
if err = cml.Init(); err != nil {
close(stop)
return
}
err = f()
if err != nil && kubeErrors.IsConflict(err) {
log.Printf("Optimistic locking conflict: trying again: %s", err)
return
}
close(stop)
}, retryPeriod, jitterFactor, true, stop)
return err
}
+245
View File
@@ -0,0 +1,245 @@
/*
Package main deals with weave-net peers on the cluster.
This involves peer management, such as getting the latest peers or removing defunct peers from the cluster
*/
package main
import (
"flag"
"fmt"
"net"
"os"
api "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
weaveapi "github.com/weaveworks/weave/api"
"github.com/weaveworks/weave/common"
)
type nodeInfo struct {
name string
addr string
}
// return the IP addresses of all nodes in the cluster
func getKubePeers(c *kubernetes.Clientset) ([]nodeInfo, error) {
nodeList, err := c.CoreV1().Nodes().List(api.ListOptions{})
if err != nil {
return nil, err
}
addresses := make([]nodeInfo, 0, len(nodeList.Items))
for _, peer := range nodeList.Items {
var internalIP, externalIP string
for _, addr := range peer.Status.Addresses {
// Check it's a valid ipv4 address
ip := net.ParseIP(addr.Address)
if ip == nil || ip.To4() == nil {
continue
}
if addr.Type == "InternalIP" {
internalIP = ip.To4().String()
} else if addr.Type == "ExternalIP" {
externalIP = ip.To4().String()
}
}
// Fallback for cases where a Node has an ExternalIP but no InternalIP
if internalIP != "" {
addresses = append(addresses, nodeInfo{name: peer.Name, addr: internalIP})
} else if externalIP != "" {
addresses = append(addresses, nodeInfo{name: peer.Name, addr: externalIP})
}
}
return addresses, nil
}
const (
configMapName = "weave-net"
configMapNamespace = "kube-system"
)
// update the list of all peers that have gone through this code path
func addMyselfToPeerList(cml *configMapAnnotations, c *kubernetes.Clientset, peerName, name string) (*peerList, error) {
var list *peerList
err := cml.LoopUpdate(func() error {
var err error
list, err = cml.GetPeerList()
if err != nil {
return err
}
if !list.contains(peerName) {
list.add(peerName, name)
err = cml.UpdatePeerList(*list)
if err != nil {
return err
}
}
return nil
})
return list, err
}
func checkIamInPeerList(cml *configMapAnnotations, c *kubernetes.Clientset, peerName string) (bool, error) {
if err := cml.Init(); err != nil {
return false, err
}
list, err := cml.GetPeerList()
if err != nil {
return false, err
}
common.Log.Debugf("[kube-peers] Checking peer %q against list %v", peerName, list)
return list.contains(peerName), nil
}
// For each of those peers that is no longer listed as a node by
// Kubernetes, remove it from Weave IPAM
func reclaimRemovedPeers(weave *weaveapi.Client, cml *configMapAnnotations, nodes []nodeInfo, myPeerName string) error {
for {
if err := cml.Init(); err != nil {
return err
}
// 1. Compare peers stored in the peerList against all peers reported by k8s now.
storedPeerList, err := cml.GetPeerList()
if err != nil {
return err
}
peerMap := make(map[string]peerInfo, len(storedPeerList.Peers))
for _, peer := range storedPeerList.Peers {
peerMap[peer.NodeName] = peer
}
for _, node := range nodes {
delete(peerMap, node.name)
}
common.Log.Debugln("[kube-peers] Nodes that have disappeared:", peerMap)
if len(peerMap) == 0 {
break
}
// 2. Loop for each X in the first set and not in the second - we wish to remove X from our data structures
for _, peer := range peerMap {
common.Log.Debugln("[kube-peers] Preparing to remove disappeared peer", peer)
okToRemove := false
// 3. Check if there is an existing annotation with key X
if existingAnnotation, found := cml.GetAnnotation(KubePeersPrefix + peer.PeerName); found {
common.Log.Debugln("[kube-peers] Existing annotation", existingAnnotation)
// 4. If annotation already contains my identity, ok;
if existingAnnotation == myPeerName {
okToRemove = true
}
} else {
// 5. If non-existent, write an annotation with key X and contents "my identity"
common.Log.Debugln("[kube-peers] Noting I plan to remove ", peer.PeerName)
if err := cml.UpdateAnnotation(KubePeersPrefix+peer.PeerName, myPeerName); err == nil {
okToRemove = true
} else {
common.Log.Debugln("[kube-peers] error from UpdateAnnotation: ", err)
}
}
if okToRemove {
// 6. If step 4 or 5 succeeded, rmpeer X
result, err := weave.RmPeer(peer.PeerName)
common.Log.Infof("[kube-peers] rmpeer of %s: %s", peer.PeerName, result)
if err != nil {
return err
}
cml.LoopUpdate(func() error {
// 7aa. Remove any annotations Z* that have contents X
if err := cml.RemoveAnnotationsWithValue(peer.PeerName); err != nil {
return err
}
// 7a. Remove X from peerList
storedPeerList.remove(peer.PeerName)
if err := cml.UpdatePeerList(*storedPeerList); err != nil {
return err
}
// 7b. Remove annotation with key X
return cml.RemoveAnnotation(KubePeersPrefix + peer.PeerName)
})
}
// 8. If step 5 failed due to optimistic lock conflict, stop: someone else is handling X
// Step 3-5 is to protect against two simultaneous rmpeers of X
// Step 4 is to pick up again after a restart between step 5 and step 7b
// If the peer doing the reclaim disappears between steps 5 and 7a, then someone will clean it up in step 7aa
// If peer doing the reclaim disappears forever between 7a and 7b then we get a dangling annotation
// This should be sufficiently rare that we don't care.
}
// 9. Go back to step 1 until there is no difference between the two sets
}
// Question: Should we narrow step 2 by checking against Weave Net IPAM?
// i.e. If peer X owns any address space and is marked unreachable, we want to rmpeer X
return nil
}
func main() {
var (
justReclaim bool
justCheck bool
justSetNodeStatus bool
peerName string
nodeName string
logLevel string
)
flag.BoolVar(&justReclaim, "reclaim", false, "reclaim IP space from dead peers")
flag.BoolVar(&justCheck, "check-peer-new", false, "return success if peer name is not stored in annotation")
flag.BoolVar(&justSetNodeStatus, "set-node-status", false, "set NodeNetworkUnavailable to false")
flag.StringVar(&peerName, "peer-name", "unknown", "name of this Weave Net peer")
flag.StringVar(&nodeName, "node-name", "unknown", "name of this Kubernetes node")
flag.StringVar(&logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
flag.Parse()
common.SetLogLevel(logLevel)
config, err := rest.InClusterConfig()
if err != nil {
common.Log.Fatalf("[kube-peers] Could not get cluster config: %v", err)
}
c, err := kubernetes.NewForConfig(config)
if err != nil {
common.Log.Fatalf("[kube-peers] Could not make Kubernetes connection: %v", err)
}
if justCheck {
cml := newConfigMapAnnotations(configMapNamespace, configMapName, c)
exists, err := checkIamInPeerList(cml, c, peerName)
if err != nil {
common.Log.Fatalf("[kube-peers] Could not check peer list: %v", err)
}
if exists {
os.Exit(9)
} else {
os.Exit(0)
}
}
if justSetNodeStatus {
err := setNodeNetworkUnavailableFalse(c, nodeName)
if err != nil {
common.Log.Fatalf("[kube-peers] could not set node status: %v", err)
}
return
}
peers, err := getKubePeers(c)
if err != nil {
common.Log.Fatalf("[kube-peers] Could not get peers: %v", err)
}
if justReclaim {
cml := newConfigMapAnnotations(configMapNamespace, configMapName, c)
list, err := addMyselfToPeerList(cml, c, peerName, nodeName)
if err != nil {
common.Log.Fatalf("[kube-peers] Could not update peer list: %v", err)
}
common.Log.Infoln("[kube-peers] Added myself to peer list", list)
weave := weaveapi.NewClient(os.Getenv("WEAVE_HTTP_ADDR"), common.Log)
err = reclaimRemovedPeers(weave, cml, peers, peerName)
if err != nil {
common.Log.Fatalf("[kube-peers] Error while reclaiming space: %v", err)
}
return
}
for _, node := range peers {
fmt.Println(node.addr)
}
}
+29
View File
@@ -0,0 +1,29 @@
package main
import (
"encoding/json"
"fmt"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
// based on k8s.io/kubernetes/pkg/controller/nodeipam/ipam/adapter.go
func setNodeNetworkUnavailableFalse(c *kubernetes.Clientset, nodeName string) error {
condition := v1.NodeCondition{
Type: v1.NodeNetworkUnavailable,
Status: v1.ConditionFalse,
Reason: "WeaveIsUp",
Message: "Weave pod has set this",
LastTransitionTime: metav1.Now(),
LastHeartbeatTime: metav1.Now(),
}
raw, err := json.Marshal(&[]v1.NodeCondition{condition})
if err != nil {
return err
}
patch := []byte(fmt.Sprintf(`{"status":{"conditions":%s}}`, raw))
_, err = c.CoreV1().Nodes().PatchStatus(nodeName, patch)
return err
}