Add release v1.7.1

Improve doc CI

CHANGELOG.md

@@ -1,3 +1,10 @@
+## Release 1.7.1
+
+Bug fixes and minor improvements
+
+* Added utility to recover container addresses without asking Docker #2531
+* Improved CI #2274
+
 ## Release 1.7.0
 
 Highlights:

bin/circle-dependencies-post

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -e
+
+# We need jq to analyse the PR files JSON, so install that first
+sudo apt-get -q update && sudo apt-get -q install bc jq
+
+echo "TEST_AND_PUBLISH=1" > "$STATE"
+
+if [ -n "$CI_PULL_REQUEST" -a "$CI_PULL_REQUEST" = "$CI_PULL_REQUESTS" ] ; then
+    if PR=$(echo "$CI_PULL_REQUEST" | grep -oP '(?<=^https://github.com/weaveworks/weave/pull/)[0-9]+$') ; then
+        echo "GH_PR=$PR" >> "$STATE"
+        if FILES=$(curl -s https://api.github.com/repos/weaveworks/weave/pulls/$PR/files | jq -r '.[]|.filename') ; then
+            echo "PR touches [$FILES]"
+            if ! echo "$FILES" | grep -v ^site/ ; then
+                echo "PR contains site changes only - skipping tests"
+                echo "TEST_AND_PUBLISH=" >> "$STATE"
+            else
+                echo "PR contains non-site changes - running tests"
+            fi
+        else
+            echo "Unable to get files for PR $PR" >&2
+        fi
+    else
+        echo "Unable to extract PR number from $CI_PULL_REQUEST" >&2
+    fi
+fi
+
+source "$STATE"
+
+mkdir -p $(dirname $SRCDIR) && cp -r $(pwd)/ $SRCDIR
+
+if [ -n "$TEST_AND_PUBLISH" ] ; then
+    bin/setup-circleci-secrets "$SECRET_PASSWORD"
+    cd $SRCDIR; git submodule update --init
+    if [ "$CIRCLE_NODE_INDEX" = "0" ] ; then
+        cd $SRCDIR/test; ./gce.sh make_template
+    fi
+    cd $SRCDIR/build
+    ../tools/rebuild-image weaveworks/weavebuild . Dockerfile build.sh
+    touch $SRCDIR/.build.uptodate
+    sudo chown -R ubuntu /usr/local/go
+fi

bin/circle-deploy-issues

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+
+source "$STATE"
+
+site_changes() {
+    curl -s https://api.github.com/repos/weaveworks/weave/pulls/$1/files |
+        jq -r '.[]|.status + " " + .filename' |
+        grep -oP '^(added|modified|removed) site/.*(?=\.md$)'
+}
+
+format_comment() {
+    if [ ! -d .git ] || ! BRANCH=$(git symbolic-ref --short HEAD) || [ -z "$BRANCH" ] ; then
+        echo "Could not determine branch" >&2
+        return 1
+    fi
+
+    echo "#### Documentation changes:"
+    while read ACTION PAGE; do
+        echo "$ACTION [${PAGE#site/}]($WP_DEV_URL/docs/net/${BRANCH#issues/}/${PAGE#site/}/)"
+    done | sort
+}
+
+post_comment() {
+    jq -s -R '. | tostring | { body: . }' |
+        curl -s -d @- \
+            -H "Authorization: Bearer $WP_GH_COMMENT_TOKEN" \
+            -H 'Content-Type: application/json' \
+            https://api.github.com/repos/weaveworks/weave/issues/$GH_PR/comments > /dev/null
+}
+
+cd $SRCDIR
+bin/install-wordepress v1.0.0
+bin/publish-site "$WP_DEV_URL" "$WP_DEV_USER" "$WP_DEV_PASSWORD"
+
+if [ -n "$WP_GH_COMMENT_TOKEN" -a -n "$GH_PR" ] ; then
+    echo ">>> Adding doc changelog to GitHub PR $GH_PR"
+    if CHANGES=$(site_changes $GH_PR) && [ -n "$CHANGES" ] ; then
+        echo "$CHANGES" | format_comment | post_comment
+    fi
+fi

bin/circle-deploy-master

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e
+
+source "$STATE"
+
+if [ -n "$TEST_AND_PUBLISH" ] ; then
+    docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
+    cd $SRCDIR
+    make clean-bin
+    make publish
+fi
+
+cd $SRCDIR
+bin/install-wordepress v1.0.0
+bin/publish-site "$WP_DEV_URL" "$WP_DEV_USER" "$WP_DEV_PASSWORD"

bin/circle-deploy-release

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+cd $SRCDIR
+bin/install-wordepress v1.0.0
+bin/publish-site "$WP_LIVE_URL" "$WP_LIVE_USER" "$WP_LIVE_PASSWORD"

bin/circle-teardown-pre

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e
+
+source "$STATE"
+
+if [ -n "$TEST_AND_PUBLISH" ] ; then
+    if [ "$CIRCLE_NODE_INDEX" = "0" ] ; then
+        go get github.com/mattn/goveralls
+        goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/test/profile.cov -service=circleci
+        cd $SRCDIR/test; cp coverage.* $CIRCLE_ARTIFACTS
+    fi
+fi

bin/circle-test-pre

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+source "$STATE"
+
+if [ -n "$TEST_AND_PUBLISH" ] ; then
+    cd $SRCDIR
+    make testrunner
+    make COVERAGE=true RM= all
+
+    cd $SRCDIR/test
+    ./gce.sh setup && eval $(./gce.sh hosts) && ./setup.sh
+fi

bin/circle-test-smoke

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+source "$STATE"
+
+if [ -n "$TEST_AND_PUBLISH" ] ; then
+    cd $SRCDIR/test
+    eval $(./gce.sh hosts)
+    export COVERAGE=true
+    ./run_all.sh
+fi

bin/circle-test-teardown

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+source "$STATE"
+
+if [ -n "$TEST_AND_PUBLISH" ] ; then
+    cd $SRCDIR/test
+    ./gce.sh destroy
+    test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; ./gen_coverage_reports.sh)
+fi

bin/circle-test-unit

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+source "$STATE"
+
+if [ -n "$TEST_AND_PUBLISH" ] ; then
+    cd $SRCDIR
+    make lint
+    COVERDIR=test/coverage make RM= tests
+fi

bin/install-wordepress

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [ $# -ne 1 ] ; then
+    echo USAGE: install-wordepress [COMMIT-ISH] >&2
+    exit 1
+fi
+
+go get github.com/weaveworks/wordepress
+cd /home/ubuntu/src/github.com/weaveworks/wordepress
+git checkout $1
+cd cmd/wordepress
+go get

build/Dockerfile

@@ -1,7 +1,7 @@
 FROM golang:1.5.2
 ENV GO15VENDOREXPERIMENT 1
 RUN apt-get update && apt-get install -y libpcap-dev python-requests time file
-RUN go get github.com/golang/lint/golint github.com/fzipp/gocyclo github.com/client9/misspell/cmd/misspell
+RUN go get github.com/fzipp/gocyclo github.com/client9/misspell/cmd/misspell
 RUN go clean -i net && go install -tags netgo std
 RUN go install -race -tags netgo std
 COPY build.sh /

circle.yml

@@ -12,46 +12,31 @@ machine:
     SRCDIR: /home/ubuntu/src/github.com/weaveworks/weave
     PATH: $PATH:$HOME/.local/bin
     CLOUDSDK_CORE_DISABLE_PROMPTS: 1
+    STATE: /home/ubuntu/state.env
 
 dependencies:
   cache_directories:
     - "~/docker"
   post:
-    - "sudo apt-get update && sudo apt-get install bc jq"
-    - bin/setup-circleci-secrets "$SECRET_PASSWORD"
-    - "mkdir -p $(dirname $SRCDIR) && cp -r $(pwd)/ $SRCDIR"
-    - "cd $SRCDIR; git submodule update --init"
-    - cd $SRCDIR/test; ./gce.sh make_template:
-        parallel: false
-    - "cd $SRCDIR/build; ../tools/rebuild-image weaveworks/weavebuild . Dockerfile build.sh && touch $SRCDIR/.build.uptodate"
-    - "sudo chown -R ubuntu /usr/local/go"
+    - bin/circle-dependencies-post
 
 test:
+  pre:
+    - "$SRCDIR/bin/circle-test-pre":
+        parallel: true
   override:
-    - cd $SRCDIR; make lint:
+    - "$SRCDIR/bin/circle-test-unit":
         parallel: true
-    - cd $SRCDIR; COVERDIR=test/coverage make RM= tests:
-        parallel: true
-    - cd $SRCDIR; make COVERAGE=true RM= exes all:
-        parallel: true
-    - cd $SRCDIR; make testrunner:
-        parallel: true
-    - cd $SRCDIR/test; ./gce.sh setup && eval $(./gce.sh hosts) && ./setup.sh:
-        parallel: true
-    - cd $SRCDIR/test; eval $(./gce.sh hosts); export COVERAGE=true; ./run_all.sh:
+    - "$SRCDIR/bin/circle-test-smoke":
         parallel: true
         timeout: 300
   post:
-    - cd $SRCDIR/test; ./gce.sh destroy:
-        parallel: true
-    - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; ./gen_coverage_reports.sh):
+    - "$SRCDIR/bin/circle-test-teardown":
         parallel: true
 
 teardown:
   pre:
-    - test "$CIRCLE_NODE_INDEX" != "0" || (go get github.com/mattn/goveralls && goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/test/profile.cov -service=circleci || true):
-        parallel: true
-    - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; cp coverage.* $CIRCLE_ARTIFACTS):
+    - "$SRCDIR/bin/circle-teardown-pre":
         parallel: true
 
 deployment:
@@ -59,20 +44,14 @@ deployment:
     branch: /[0-9]+\.[0-9]+/
     owner: weaveworks
     commands:
-      - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get
-      - cd $SRCDIR; bin/publish-site "$WP_LIVE_URL" "$WP_LIVE_USER" "$WP_LIVE_PASSWORD"
-  hub:
+      - "$SRCDIR/bin/circle-deploy-release"
+  master:
     branch: master
     owner: weaveworks
     commands:
-      - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
-      - cd $SRCDIR; make clean-bin
-      - cd $SRCDIR; make exes publish
-#      - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get
-#      - cd $SRCDIR; bin/publish-site "$WP_DEV_URL" "$WP_DEV_USER" "$WP_DEV_PASSWORD"
-#  issues:
-#    branch: /.*/
-#    owner: weaveworks
-#    commands:
-#      - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get
-#      - cd $SRCDIR; bin/publish-site "$WP_DEV_URL" "$WP_DEV_USER" "$WP_DEV_PASSWORD"
+      - "$SRCDIR/bin/circle-deploy-master"
+  issues:
+    branch: /.*/
+    owner: weaveworks
+    commands:
+      - "$SRCDIR/bin/circle-deploy-issues"

common/process.go

@@ -0,0 +1,27 @@
+package common
+
+import (
+	"os"
+	"strconv"
+)
+
+func AllPids(procRoot string) ([]int, error) {
+	fh, err := os.Open(procRoot)
+	if err != nil {
+		return nil, err
+	}
+	defer fh.Close()
+	dirNames, err := fh.Readdirnames(-1)
+	if err != nil {
+		return nil, err
+	}
+	pids := make([]int, len(dirNames))
+	for _, dirName := range dirNames {
+		pid, err := strconv.Atoi(dirName)
+		if err != nil { // Only interested in numeric entries - skip /proc/stat, etc.
+			continue
+		}
+		pids = append(pids, pid)
+	}
+	return pids, err
+}

common/utils.go

@@ -125,7 +125,7 @@ func ConnectedToBridgePredicate(bridgeName string) (func(link netlink.Link) bool
 }
 
 func GetNetDevsWithPredicate(processID int, predicate func(link netlink.Link) bool) ([]NetDev, error) {
-	// Bail out if this container is running in the root namespace
+	// Bail out if this process is running in the root namespace
 	nsToplevel, err := netns.GetFromPid(1)
 	if err != nil {
 		return nil, fmt.Errorf("unable to open root namespace: %s", err)
@@ -133,6 +133,7 @@ func GetNetDevsWithPredicate(processID int, predicate func(link netlink.Link) bo
 	defer nsToplevel.Close()
 	nsContainr, err := netns.GetFromPid(processID)
 	if err != nil {
+		// Unable to find a namespace for this process - just return nothing
 		if os.IsNotExist(err) {
 			return nil, nil
 		}

prog/weaveutil/docker_tls_args.go

@@ -8,6 +8,8 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+
+	"github.com/weaveworks/weave/common"
 )
 
 func dockerTLSArgs(args []string) error {
@@ -15,18 +17,14 @@ func dockerTLSArgs(args []string) error {
 		cmdUsage("docker-tls-args", "")
 	}
 	procRoot := "/proc"
-	dirEntries, err := ioutil.ReadDir(procRoot)
+	pids, err := common.AllPids(procRoot)
 	if err != nil {
 		return err
 	}
 
-	for _, dirEntry := range dirEntries {
-		dirName := dirEntry.Name()
-		if _, err := strconv.Atoi(dirName); err != nil {
-			continue
-		}
-
+	for _, pid := range pids {
 		isDaemon := false
+		dirName := strconv.Itoa(pid)
 		if comm, err := ioutil.ReadFile(filepath.Join(procRoot, dirName, "comm")); err != nil {
 			continue
 		} else if string(comm) == "dockerd\n" {

prog/weaveutil/main.go

@@ -20,6 +20,7 @@ func init() {
 		"create-plugin-network":  createPluginNetwork,
 		"remove-plugin-network":  removePluginNetwork,
 		"container-addrs":        containerAddrs,
+		"process-addrs":          processAddrs,
 		"attach-container":       attach,
 		"detach-container":       detach,
 	}

prog/weaveutil/proc-addrs.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"strconv"
+
+	"github.com/weaveworks/weave/common"
+	weavenet "github.com/weaveworks/weave/net"
+)
+
+func processAddrs(args []string) error {
+	if len(args) < 1 {
+		cmdUsage("process-addrs", "<bridgeName>")
+	}
+	bridgeName := args[0]
+
+	pred, err := common.ConnectedToBridgePredicate(bridgeName)
+	if err != nil {
+		if err == weavenet.ErrLinkNotFound {
+			return nil
+		}
+		return err
+	}
+
+	pids, err := common.AllPids("/proc")
+	if err != nil {
+		return err
+	}
+
+	// NB: Because network namespaces (netns) are changed many times inside the loop,
+	//     it's NOT safe to exec any code depending on the root netns without
+	//     wrapping with WithNetNS*.
+	for _, pid := range pids {
+		netDevs, err := common.GetNetDevsWithPredicate(pid, pred)
+		if err != nil {
+			return err
+		}
+		printNetDevs(strconv.Itoa(pid), netDevs)
+	}
+	return nil
+}