Adds a CANARY chain and Monitor loop instead of reapplying iptables

This commit is contained in:
Simon Howe committed 2020-07-15 11:37:16 +02:00
1 parent 94e75e45af
commit 66eb80d841
5 files changed
+135 -47

No files matched your search

+2 -7
View File
@@ -103,13 +103,8 @@ func addNatRule(ipt *iptables.IPTables, rulespec ...string) error {
// Loop until we get an exit code other than "temporarily unavailable"
for {
if err := ipt.AppendUnique("nat", "WEAVE", rulespec...); err != nil {
if ierr, ok := err.(*iptables.Error); ok {
if status, ok := ierr.ExitError.Sys().(syscall.WaitStatus); ok {
// (magic exit code 4 found in iptables source code; undocumented)
if status.ExitStatus() == 4 {
continue
}
}
if isResourceError(err) {
continue
}
return err
}
+118
View File
@@ -2,9 +2,13 @@ package net
import (
"strings"
"syscall"
"time"
"github.com/coreos/go-iptables/iptables"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
utilwait "k8s.io/apimachinery/pkg/util/wait"
)
// AddChainWithRules creates a chain and appends given rules to it.
@@ -104,3 +108,117 @@ func ensureRulesAtTop(table, chain string, rulespecs [][]string, ipt *iptables.I
return nil
}
func chainExists(ipt *iptables.IPTables, table string, chain string) (bool, error) {
existingChains, err := ipt.ListChains(table)
if err != nil {
return false, errors.Wrapf(err, "ipt.ListChains(%s)", table)
}
chainMap := make(map[string]struct{})
for _, c := range existingChains {
chainMap[c] = struct{}{}
}
_, found := chainMap[chain]
return found, nil
}
const (
// Max time we wait for an iptables flush to complete after we notice it has started
iptablesFlushTimeout = 5 * time.Second
// How often we poll while waiting for an iptables flush to complete
iptablesFlushPollTime = 100 * time.Millisecond
)
// Monitor is part of Interface
func Monitor(log *logrus.Logger, canary string, tables []string, reloadFunc func(), interval time.Duration, stopCh <-chan struct{}) {
ipt, err := iptables.New()
if err != nil {
log.Errorf("creating iptables object while initializing iptable Monitoring %s", err)
return
}
for {
_ = PollImmediateUntil(interval, func() (bool, error) {
for _, table := range tables {
if err := ensureChains(ipt, table, canary); err != nil {
log.Warningf("Could not set up iptables canary %s/%s: %v", table, canary, err)
return false, nil
}
}
return true, nil
}, stopCh)
// Poll until stopCh is closed or iptables is flushed
err = utilwait.PollUntil(interval, func() (bool, error) {
if exists, err := chainExists(ipt, tables[0], canary); exists {
return false, nil
} else if isResourceError(err) {
log.Warningf("Could not check for iptables canary %s/%s: %v", tables[0], canary, err)
return false, nil
}
log.Infof("iptables canary %s/%s deleted", tables[0], canary)
// Wait for the other canaries to be deleted too before returning
// so we don't start reloading too soon.
err := utilwait.PollImmediate(iptablesFlushPollTime, iptablesFlushTimeout, func() (bool, error) {
for i := 1; i < len(tables); i++ {
if exists, err := chainExists(ipt, tables[i], canary); exists || isResourceError(err) {
return false, nil
}
}
return true, nil
})
if err != nil {
log.Warning("Inconsistent iptables state detected.")
}
return true, nil
}, stopCh)
if err != nil {
// stopCh was closed
for _, table := range tables {
_ = ipt.DeleteChain(table, canary)
}
return
}
log.Infof("Reloading after iptables flush")
reloadFunc()
}
}
const iptablesStatusResourceProblem = 4
// isResourceError returns true if the error indicates that iptables ran into a "resource
// problem" and was unable to attempt the request. In particular, this will be true if it
// times out trying to get the iptables lock.
func isResourceError(err error) bool {
if ierr, ok := err.(*iptables.Error); ok {
if status, ok := ierr.ExitError.Sys().(syscall.WaitStatus); ok {
return status.ExitStatus() == iptablesStatusResourceProblem
}
}
return false
}
// PollImmediateUntil tries a condition func until it returns true, an error or stopCh is closed.
//
// PollImmediateUntil runs the 'condition' before waiting for the interval.
// 'condition' will always be invoked at least once.
func PollImmediateUntil(interval time.Duration, condition utilwait.ConditionFunc, stopCh <-chan struct{}) error {
done, err := condition()
if err != nil {
return err
}
if done {
return nil
}
select {
case <-stopCh:
return utilwait.ErrWaitTimeout
default:
return utilwait.PollUntil(interval, condition, stopCh)
}
}