Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5232a9af6b | ||
|
|
c393b23b31 | ||
|
|
955fbca44a | ||
|
|
18c26e9fbb | ||
|
|
b54efbde69 | ||
|
|
be9d64b2cc | ||
|
|
c6f356a167 | ||
|
|
eae45ebc94 | ||
|
|
ea50b3d5cc | ||
|
|
3297b93edb | ||
|
|
6d0e867ad0 | ||
|
|
1cea9c1c87 | ||
|
|
3320a9e7d4 | ||
|
|
9b90a938f5 | ||
|
|
c6d0149ba6 | ||
|
|
598efe5d1a | ||
|
|
3679065258 | ||
|
|
4649f1158a | ||
|
|
de17a44f35 | ||
|
|
52ce4df18e | ||
|
|
f1ed5bf838 | ||
|
|
256a801d16 | ||
|
|
c416cf8da5 | ||
|
|
75e812a061 | ||
|
|
4cc7b01940 | ||
|
|
a2ff7e5981 | ||
|
|
cda98a6375 | ||
|
|
3b7a754b0a | ||
|
|
c01c6bcb91 | ||
|
|
e47360f8dc | ||
|
|
6035b54277 | ||
|
|
a238130613 | ||
|
|
998be42a52 | ||
|
|
14944da396 | ||
|
|
a3974c2aeb | ||
|
|
13ec517f94 | ||
|
|
43419815f4 | ||
|
|
b5256f7b71 | ||
|
|
0caa9dd652 | ||
|
|
5e1e8ec543 |
No files matched your search
@@ -1,3 +1,28 @@
|
|||||||
|
## Release 1.4.3
|
||||||
|
|
||||||
|
Preparing for Docker 1.10, plus some bug-fixes.
|
||||||
|
|
||||||
|
* Avoid hang in Docker v1.10 on `docker volume ls` after `weave stop`,
|
||||||
|
`weave stop-plugin` or `weave reset` #1934/#1936
|
||||||
|
* Fix "unexpected EOF" from Docker 1.10 on `docker exec` with Weave
|
||||||
|
proxy `--rewrite-inspect` enabled #1911/#1917
|
||||||
|
* Avoid losing DNS entries and potentially double-allocating IP
|
||||||
|
Addresses allocated via plugin, on router restart; also extend
|
||||||
|
`weave ps` to show IP addresses allocated via plugin #1745/#1921
|
||||||
|
* Stop creating lots of copies of `weavewait` program in Docker
|
||||||
|
volumes #1757/#1935
|
||||||
|
* Prevent container starting prematurely when proxy in
|
||||||
|
`--no-multicast-route` mode #1942/#1943
|
||||||
|
* Log error message from plugin rather than crashing when weave not
|
||||||
|
running #1906/#1918
|
||||||
|
* Warn, don't error, if unable to remove plugin network in 'weave
|
||||||
|
stop', to avoid breaking the usual upgrade or config change process
|
||||||
|
#1900/#1919
|
||||||
|
* Don't crash if network conditions suggest only very small packets
|
||||||
|
will get through #1905/#1926
|
||||||
|
* Cope with unexpected errors during route traversal when starting
|
||||||
|
container via proxy #1909/#1910/#1932
|
||||||
|
|
||||||
## Release 1.4.2
|
## Release 1.4.2
|
||||||
|
|
||||||
Bug-fixes and minor improvements.
|
Bug-fixes and minor improvements.
|
||||||
|
|||||||
@@ -119,7 +119,7 @@ $(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile prog/weaveexec/symlink $(DOCKER
|
|||||||
$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -t $(WEAVEEXEC_IMAGE) prog/weaveexec
|
$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -t $(WEAVEEXEC_IMAGE) prog/weaveexec
|
||||||
touch $@
|
touch $@
|
||||||
|
|
||||||
$(DOCKERPLUGIN_UPTODATE): prog/plugin/Dockerfile $(DOCKERPLUGIN_EXE)
|
$(DOCKERPLUGIN_UPTODATE): prog/plugin/Dockerfile $(DOCKERPLUGIN_EXE) $(WEAVEEXEC_UPTODATE)
|
||||||
$(SUDO) docker build -t $(DOCKERPLUGIN_IMAGE) prog/plugin
|
$(SUDO) docker build -t $(DOCKERPLUGIN_IMAGE) prog/plugin
|
||||||
touch $@
|
touch $@
|
||||||
|
|
||||||
|
|||||||
@@ -52,16 +52,25 @@ func EnsureInterfaceAndMcastRoute(ifaceName string) (*net.Interface, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
dest := net.IPv4(224, 0, 0, 0)
|
dest := net.IPv4(224, 0, 0, 0)
|
||||||
// check for currently-existing route after subscribing, to avoid race
|
check := func(route netlink.Route) bool {
|
||||||
if CheckRouteExists(ifaceName, dest) {
|
return route.LinkIndex == iface.Index && route.Dst != nil && route.Dst.IP.Equal(dest)
|
||||||
return iface, err
|
|
||||||
}
|
}
|
||||||
for update := range ch {
|
// check for currently-existing route after subscribing, to avoid race
|
||||||
link, _ := netlink.LinkByIndex(update.Route.LinkIndex)
|
routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
|
||||||
if link.Attrs().Name == ifaceName && update.Route.Dst.IP.Equal(dest) {
|
if err != nil {
|
||||||
break
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, route := range routes {
|
||||||
|
if check(route) {
|
||||||
|
return iface, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
for update := range ch {
|
||||||
|
if check(update.Route) {
|
||||||
|
return iface, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// should never get here
|
||||||
return iface, nil
|
return iface, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+12
-20
@@ -10,7 +10,7 @@ import (
|
|||||||
// A network is considered free if it does not overlap any existing
|
// A network is considered free if it does not overlap any existing
|
||||||
// routes on this host. This is the same approach taken by Docker.
|
// routes on this host. This is the same approach taken by Docker.
|
||||||
func CheckNetworkFree(subnet *net.IPNet, ignoreIfaceNames map[string]struct{}) error {
|
func CheckNetworkFree(subnet *net.IPNet, ignoreIfaceNames map[string]struct{}) error {
|
||||||
return forEachRoute(ignoreIfaceNames, func(name string, route netlink.Route) error {
|
return forEachRoute(ignoreIfaceNames, func(route netlink.Route) error {
|
||||||
if route.Dst != nil && overlaps(route.Dst, subnet) {
|
if route.Dst != nil && overlaps(route.Dst, subnet) {
|
||||||
return fmt.Errorf("Network %s overlaps with existing route %s on host.", subnet, route.Dst)
|
return fmt.Errorf("Network %s overlaps with existing route %s on host.", subnet, route.Dst)
|
||||||
}
|
}
|
||||||
@@ -26,7 +26,7 @@ func overlaps(n1, n2 *net.IPNet) bool {
|
|||||||
// For a specific address, we only care if it is actually *inside* an
|
// For a specific address, we only care if it is actually *inside* an
|
||||||
// existing route, because weave-local traffic never hits IP routing.
|
// existing route, because weave-local traffic never hits IP routing.
|
||||||
func CheckAddressOverlap(addr net.IP, ignoreIfaceNames map[string]struct{}) error {
|
func CheckAddressOverlap(addr net.IP, ignoreIfaceNames map[string]struct{}) error {
|
||||||
return forEachRoute(ignoreIfaceNames, func(name string, route netlink.Route) error {
|
return forEachRoute(ignoreIfaceNames, func(route netlink.Route) error {
|
||||||
if route.Dst != nil && route.Dst.Contains(addr) {
|
if route.Dst != nil && route.Dst.Contains(addr) {
|
||||||
return fmt.Errorf("Address %s overlaps with existing route %s on host.", addr, route.Dst)
|
return fmt.Errorf("Address %s overlaps with existing route %s on host.", addr, route.Dst)
|
||||||
}
|
}
|
||||||
@@ -34,32 +34,24 @@ func CheckAddressOverlap(addr net.IP, ignoreIfaceNames map[string]struct{}) erro
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func forEachRoute(ignoreIfaceNames map[string]struct{}, check func(name string, r netlink.Route) error) error {
|
func forEachRoute(ignoreIfaceNames map[string]struct{}, check func(r netlink.Route) error) error {
|
||||||
|
ignoreIfaceIndices := make(map[int]struct{})
|
||||||
|
for ifaceName := range ignoreIfaceNames {
|
||||||
|
if iface, err := net.InterfaceByName(ifaceName); err == nil {
|
||||||
|
ignoreIfaceIndices[iface.Index] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
|
routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
for _, route := range routes {
|
for _, route := range routes {
|
||||||
link, err := netlink.LinkByIndex(route.LinkIndex)
|
if _, found := ignoreIfaceIndices[route.LinkIndex]; found {
|
||||||
if err == nil {
|
continue
|
||||||
if _, found := ignoreIfaceNames[link.Attrs().Name]; found {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if err := check(link.Attrs().Name, route); err != nil {
|
if err := check(route); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func CheckRouteExists(ifaceName string, dest net.IP) bool {
|
|
||||||
found := false
|
|
||||||
forEachRoute(map[string]struct{}{}, func(name string, route netlink.Route) error {
|
|
||||||
if name == ifaceName && route.Dst.IP.Equal(dest) {
|
|
||||||
found = true
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
return found
|
|
||||||
}
|
|
||||||
+12
-1
@@ -1,10 +1,12 @@
|
|||||||
package ipamplugin
|
package ipamplugin
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"net"
|
"net"
|
||||||
|
|
||||||
"github.com/docker/libnetwork/ipamapi"
|
"github.com/docker/libnetwork/ipamapi"
|
||||||
"github.com/docker/libnetwork/netlabel"
|
"github.com/docker/libnetwork/netlabel"
|
||||||
|
godocker "github.com/fsouza/go-dockerclient"
|
||||||
"github.com/weaveworks/weave/api"
|
"github.com/weaveworks/weave/api"
|
||||||
. "github.com/weaveworks/weave/common"
|
. "github.com/weaveworks/weave/common"
|
||||||
"github.com/weaveworks/weave/common/docker"
|
"github.com/weaveworks/weave/common/docker"
|
||||||
@@ -20,7 +22,13 @@ type ipam struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewIpam(client *docker.Client, version string) (ipamapi.Ipam, error) {
|
func NewIpam(client *docker.Client, version string) (ipamapi.Ipam, error) {
|
||||||
resolver := func() (string, error) { return client.GetContainerIP(WeaveContainer) }
|
resolver := func() (string, error) {
|
||||||
|
addr, err := client.GetContainerIP(WeaveContainer)
|
||||||
|
if _, ok := err.(*godocker.NoSuchContainer); ok {
|
||||||
|
return "", fmt.Errorf("%s container is not present. Have you launched it?", WeaveContainer)
|
||||||
|
}
|
||||||
|
return addr, err
|
||||||
|
}
|
||||||
return &ipam{client: client, weave: api.NewClientWithResolver(resolver)}, nil
|
return &ipam{client: client, weave: api.NewClientWithResolver(resolver)}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -33,6 +41,9 @@ func (i *ipam) RequestPool(addressSpace, pool, subPool string, options map[strin
|
|||||||
Log.Debugln("RequestPool", addressSpace, pool, subPool, options)
|
Log.Debugln("RequestPool", addressSpace, pool, subPool, options)
|
||||||
cidr, err := i.weave.DefaultSubnet()
|
cidr, err := i.weave.DefaultSubnet()
|
||||||
Log.Debugln("RequestPool returning ", cidr, err)
|
Log.Debugln("RequestPool returning ", cidr, err)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, nil, err
|
||||||
|
}
|
||||||
// Pass back a fake "gateway address"; we don't actually use it,
|
// Pass back a fake "gateway address"; we don't actually use it,
|
||||||
// so just give the network address.
|
// so just give the network address.
|
||||||
data := map[string]string{netlabel.Gateway: cidr.String()}
|
data := map[string]string{netlabel.Gateway: cidr.String()}
|
||||||
|
|||||||
@@ -265,6 +265,7 @@ func sendError(w http.ResponseWriter, msg string, code int) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func errorResponse(w http.ResponseWriter, fmtString string, item ...interface{}) {
|
func errorResponse(w http.ResponseWriter, fmtString string, item ...interface{}) {
|
||||||
|
w.WriteHeader(http.StatusInternalServerError)
|
||||||
json.NewEncoder(w).Encode(map[string]string{
|
json.NewEncoder(w).Encode(map[string]string{
|
||||||
"Err": fmt.Sprintf(fmtString, item...),
|
"Err": fmt.Sprintf(fmtString, item...),
|
||||||
})
|
})
|
||||||
|
|||||||
+21
-14
@@ -69,25 +69,35 @@ func main() {
|
|||||||
Log.Println("Weave plugin", version, "Command line options:", os.Args[1:])
|
Log.Println("Weave plugin", version, "Command line options:", os.Args[1:])
|
||||||
Log.Info(dockerClient.Info())
|
Log.Info(dockerClient.Info())
|
||||||
|
|
||||||
var globalListener, meshListener net.Listener
|
err = run(dockerClient, address, meshAddress, meshNetworkName, nameserver, noMulticastRoute)
|
||||||
|
if err != nil {
|
||||||
|
Log.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func run(dockerClient *docker.Client, address, meshAddress, meshNetworkName, nameserver string, noMulticastRoute bool) error {
|
||||||
endChan := make(chan error, 1)
|
endChan := make(chan error, 1)
|
||||||
if address != "" {
|
if address != "" {
|
||||||
globalListener, err := listenAndServe(dockerClient, address, nameserver, noMulticastRoute, endChan, "global", false)
|
globalListener, err := listenAndServe(dockerClient, address, nameserver, noMulticastRoute, endChan, "global", false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Log.Fatalf("unable to create driver: %s", err)
|
return err
|
||||||
}
|
}
|
||||||
|
defer os.Remove(address)
|
||||||
defer globalListener.Close()
|
defer globalListener.Close()
|
||||||
}
|
}
|
||||||
if meshAddress != "" {
|
if meshAddress != "" {
|
||||||
meshListener, err := listenAndServe(dockerClient, meshAddress, nameserver, noMulticastRoute, endChan, "local", true)
|
meshListener, err := listenAndServe(dockerClient, meshAddress, nameserver, noMulticastRoute, endChan, "local", true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Log.Fatalf("unable to create driver: %s", err)
|
return err
|
||||||
}
|
}
|
||||||
|
defer os.Remove(meshAddress)
|
||||||
defer meshListener.Close()
|
defer meshListener.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
if meshNetworkName != "" {
|
if meshNetworkName != "" {
|
||||||
createNetwork(dockerClient, meshNetworkName, meshAddress)
|
if err := createNetwork(dockerClient, meshNetworkName, meshAddress); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
sigChan := make(chan os.Signal, 1)
|
sigChan := make(chan os.Signal, 1)
|
||||||
@@ -96,16 +106,12 @@ func main() {
|
|||||||
select {
|
select {
|
||||||
case sig := <-sigChan:
|
case sig := <-sigChan:
|
||||||
Log.Debugf("Caught signal %s; shutting down", sig)
|
Log.Debugf("Caught signal %s; shutting down", sig)
|
||||||
if err := dockerClient.Client.RemoveNetwork(meshNetworkName); err != nil {
|
if meshNetworkName != "" {
|
||||||
Log.Fatal(err)
|
return dockerClient.Client.RemoveNetwork(meshNetworkName)
|
||||||
}
|
}
|
||||||
|
return nil
|
||||||
case err := <-endChan:
|
case err := <-endChan:
|
||||||
if err != nil {
|
return err
|
||||||
Log.Errorf("Error from listener: %s", err)
|
|
||||||
globalListener.Close()
|
|
||||||
meshListener.Close()
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -141,7 +147,7 @@ func listenAndServe(dockerClient *docker.Client, address, nameserver string, noM
|
|||||||
return listener, nil
|
return listener, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func createNetwork(dockerClient *docker.Client, networkName, address string) {
|
func createNetwork(dockerClient *docker.Client, networkName, address string) error {
|
||||||
if _, err := dockerClient.Client.NetworkInfo(networkName); err == nil {
|
if _, err := dockerClient.Client.NetworkInfo(networkName); err == nil {
|
||||||
Log.Printf("Docker network '%s' already exists", networkName)
|
Log.Printf("Docker network '%s' already exists", networkName)
|
||||||
} else if _, ok := err.(*go_docker.NoSuchNetwork); ok {
|
} else if _, ok := err.(*go_docker.NoSuchNetwork); ok {
|
||||||
@@ -157,7 +163,8 @@ func createNetwork(dockerClient *docker.Client, networkName, address string) {
|
|||||||
}
|
}
|
||||||
_, err := dockerClient.Client.CreateNetwork(options)
|
_, err := dockerClient.Client.CreateNetwork(options)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Log.Fatalf("Error creating network: %s", err)
|
return fmt.Errorf("Error creating network: %s", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
@@ -4,9 +4,6 @@ MAINTAINER Weaveworks Inc <help@weave.works>
|
|||||||
LABEL works.weave.role=system
|
LABEL works.weave.role=system
|
||||||
|
|
||||||
WORKDIR /home/weave
|
WORKDIR /home/weave
|
||||||
VOLUME /w
|
|
||||||
VOLUME /w-noop
|
|
||||||
VOLUME /w-nomcast
|
|
||||||
ENTRYPOINT ["/home/weave/sigproxy", "/home/weave/weave"]
|
ENTRYPOINT ["/home/weave/sigproxy", "/home/weave/weave"]
|
||||||
|
|
||||||
RUN apk add --update \
|
RUN apk add --update \
|
||||||
|
|||||||
@@ -22,6 +22,10 @@ func (i *inspectExecInterceptor) InterceptResponse(r *http.Response) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if _, ok := exec["Container"]; !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
container, err := exec.Object("Container")
|
container, err := exec.Object("Container")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
+1
-1
@@ -917,7 +917,7 @@ func (fwd *sleeveForwarder) processSendError(err error) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
fwd.mtuHighestGood = 8
|
fwd.mtuHighestGood = 552
|
||||||
fwd.mtuLowestBad = mtu + 1
|
fwd.mtuLowestBad = mtu + 1
|
||||||
fwd.mtuCandidate = mtu
|
fwd.mtuCandidate = mtu
|
||||||
fwd.mtuTestsSent = 0
|
fwd.mtuTestsSent = 0
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
<div class="navbar navbar-inverse">
|
<div class="navbar navbar-inverse">
|
||||||
<div class="container">
|
<div class="container">
|
||||||
<div class="navbar-header">
|
<div class="navbar-header">
|
||||||
<a class="navbar-brand" href="http://weave.works/"><img src="http://weave.works/img/weave-logo.png"></a>
|
<a class="navbar-brand" href="http://weave.works/"><img src="http://www.weave.works/wp-content/uploads/2015/11/weave-logo-nav.png" class=" img-responsive" align="left"></a>
|
||||||
</div>
|
</div>
|
||||||
<div class="nav-wrapper">
|
<div class="nav-wrapper">
|
||||||
<ul class="nav navbar-nav navbar-right">
|
<ul class="nav navbar-nav navbar-right">
|
||||||
|
|||||||
+27
-24
@@ -5,36 +5,33 @@ layout: default
|
|||||||
|
|
||||||
# Weave Plugin
|
# Weave Plugin
|
||||||
|
|
||||||
New in Docker version 1.9 is a plugin mechanism to add different
|
Docker versions 1.9 and later have a plugin mechanism for adding
|
||||||
network providers.
|
different network providers. Weave installs itself as a network plugin
|
||||||
|
when you start it with `weave launch`. To create a network that spans
|
||||||
|
multiple docker hosts, the weave peers must be connected in the usual
|
||||||
|
way, i.e. by specifying other hosts in `weave launch` or
|
||||||
|
[`weave connect`](features.html#dynamic-topologies).
|
||||||
|
|
||||||
To install, download the `weave` script as per the [readme][].
|
Subsequently you can start containers with, e.g.
|
||||||
|
|
||||||
The Weave Net plugin runs automatically when you `weave launch`. You
|
|
||||||
must still tell the weave peers to connect to one another, either via
|
|
||||||
`weave launch` or `weave connect`.
|
|
||||||
|
|
||||||
## Starting a container
|
|
||||||
|
|
||||||
$ docker run --net=weave -ti ubuntu
|
$ docker run --net=weave -ti ubuntu
|
||||||
|
|
||||||
## Using WeaveDNS for Service Discovery
|
on any of the hosts, and they can all communicate with each other.
|
||||||
|
|
||||||
You need to pass the additional arguments `--dns` and `-dns-search`,
|
In order to use Weave's [Service Discovery](weavedns.html), you
|
||||||
for which we provide a helper in the weave script:
|
need to pass the additional arguments `--dns` and `-dns-search`, for
|
||||||
|
which we provide a helper in the weave script:
|
||||||
|
|
||||||
$ docker run --net=weave -h foo.weave.local $(weave dns-args) -tdi ubuntu
|
$ docker run --net=weave -h foo.weave.local $(weave dns-args) -tdi ubuntu
|
||||||
$ docker run --net=weave -h bar.weave.local $(weave dns-args) -ti ubuntu
|
$ docker run --net=weave -h bar.weave.local $(weave dns-args) -ti ubuntu
|
||||||
# ping foo
|
# ping foo
|
||||||
|
|
||||||
For more details see the [Weave Service Discovery documentation][service-discovery].
|
|
||||||
|
|
||||||
## Under the hood
|
## Under the hood
|
||||||
|
|
||||||
The Weave Net plugin actually provides *two* network drivers to Docker
|
The Weave plugin actually provides *two* network drivers to Docker
|
||||||
- one named `weavemesh` that can operate without a [cluster
|
- one named `weavemesh` that can operate without a cluster store and
|
||||||
store](#cluster-store) (like Docker's overlay driver) and one named
|
one named `weave` that can only work with one (like Docker's overlay
|
||||||
`weave` that can only work with one.
|
driver).
|
||||||
|
|
||||||
### `weavemesh` driver
|
### `weavemesh` driver
|
||||||
|
|
||||||
@@ -54,7 +51,8 @@ networks.
|
|||||||
* Used with Docker's cluster store based IPAM
|
* Used with Docker's cluster store based IPAM
|
||||||
|
|
||||||
There's no specific documentation from Docker on using a cluster
|
There's no specific documentation from Docker on using a cluster
|
||||||
store, but the first part of [Getting Started with Docker Multi-host Networking][docker-net]
|
store, but the first part of
|
||||||
|
[Getting Started with Docker Multi-host Networking](https://github.com/docker/docker/blob/master/docs/userguide/networking/get-started-overlay.md)
|
||||||
should point the way.
|
should point the way.
|
||||||
|
|
||||||
Note that in the case of multiple networks using the `weave` driver, all containers are
|
Note that in the case of multiple networks using the `weave` driver, all containers are
|
||||||
@@ -87,9 +85,14 @@ route, add the `--no-multicast-route` flag to `weave launch-plugin`.
|
|||||||
We start the plugin with a policy of `--restart=always`, so that it is
|
We start the plugin with a policy of `--restart=always`, so that it is
|
||||||
there after a restart or reboot. If you remove this container
|
there after a restart or reboot. If you remove this container
|
||||||
(e.g. using `weave reset`) before removing all endpoints created using
|
(e.g. using `weave reset`) before removing all endpoints created using
|
||||||
`--net=weave`, Docker can
|
`--net=weave`, Docker may hang for a long time when it subsequently
|
||||||
[hang](https://github.com/docker/libnetwork/issues/813).
|
tries to talk to the plugin.
|
||||||
|
|
||||||
[readme]: https://github.com/weaveworks/weave/blob/master/README.md#installation
|
Unfortunately, [Docker 1.9 may also try to talk to the plugin before it has even started it](https://github.com/docker/libnetwork/issues/813).
|
||||||
[service-discovery]: weavedns.html
|
If using `systemd`, we advise that you modify the Docker unit to
|
||||||
[docker-net]: https://github.com/docker/docker/blob/master/docs/userguide/networking/get-started-overlay.md
|
remove the timeout on startup, to give Docker enough time to abandon
|
||||||
|
its attempts.
|
||||||
|
|
||||||
|
E.g. in the file `/lib/systemd/system/docker.service`, add under `[Service]`:
|
||||||
|
|
||||||
|
TimeoutStartSec=0
|
||||||
@@ -8,7 +8,8 @@ C2=10.2.1.71
|
|||||||
# 700 = Linux's minimum PMTU (552) + weave framing overheads
|
# 700 = Linux's minimum PMTU (552) + weave framing overheads
|
||||||
# (currently about 60, but we leave some margin). That way the PMTUs
|
# (currently about 60, but we leave some margin). That way the PMTUs
|
||||||
# seen by containers is still >= 552.
|
# seen by containers is still >= 552.
|
||||||
drop_large_udp_packets="FORWARD -p udp --destination-port 6783 -m length '!' --length 0:700 -j DROP"
|
drop_large_sleeve_packets="INPUT -p udp --destination-port 6783 -m length '!' --length 0:700 -j DROP"
|
||||||
|
drop_large_fastdp_packets="INPUT -p udp --destination-port 6784 -m length '!' --length 0:700 -j DROP"
|
||||||
|
|
||||||
start_suite "PMTU discovery and packet fragmentation"
|
start_suite "PMTU discovery and packet fragmentation"
|
||||||
|
|
||||||
@@ -18,7 +19,8 @@ weave_on $HOST2 launch
|
|||||||
start_container $HOST1 $C1/24 --name=c1
|
start_container $HOST1 $C1/24 --name=c1
|
||||||
start_container $HOST2 $C2/24 --name=c2
|
start_container $HOST2 $C2/24 --name=c2
|
||||||
|
|
||||||
run_on $HOST2 "sudo iptables -I $drop_large_udp_packets"
|
run_on $HOST1 "sudo iptables -I $drop_large_sleeve_packets"
|
||||||
|
run_on $HOST1 "sudo iptables -I $drop_large_fastdp_packets"
|
||||||
assert_raises "weave_on $HOST2 connect $HOST1"
|
assert_raises "weave_on $HOST2 connect $HOST1"
|
||||||
sleep 35 # give weave time to discover the PMTU
|
sleep 35 # give weave time to discover the PMTU
|
||||||
|
|
||||||
@@ -28,6 +30,7 @@ sleep 35 # give weave time to discover the PMTU
|
|||||||
exec_on $HOST2 c2 $PING -s 10000 $C1 1>/dev/null 2>&1 || true
|
exec_on $HOST2 c2 $PING -s 10000 $C1 1>/dev/null 2>&1 || true
|
||||||
assert_raises "exec_on $HOST2 c2 $PING -s 10000 $C1"
|
assert_raises "exec_on $HOST2 c2 $PING -s 10000 $C1"
|
||||||
|
|
||||||
run_on $HOST2 "sudo iptables -D $drop_large_udp_packets"
|
run_on $HOST1 "sudo iptables -D $drop_large_fastdp_packets"
|
||||||
|
run_on $HOST1 "sudo iptables -D $drop_large_sleeve_packets"
|
||||||
|
|
||||||
end_suite
|
end_suite
|
||||||
@@ -6,15 +6,26 @@ entrypoint() {
|
|||||||
docker_on $HOST1 inspect --format="{{.Config.Entrypoint}}" "$@"
|
docker_on $HOST1 inspect --format="{{.Config.Entrypoint}}" "$@"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
check_iface_ready() {
|
||||||
|
assert_raises "proxy docker_on $HOST1 run -e 'WEAVE_CIDR=$1' $BASE_IMAGE $CHECK_ETHWE_UP"
|
||||||
|
assert_raises "proxy docker_on $HOST1 run -e 'WEAVE_CIDR=$1' $BASE_IMAGE ip -4 -o addr show ethwe | grep -q '$1'"
|
||||||
|
}
|
||||||
|
|
||||||
start_suite "Proxy waits for weave to be ready before running container commands"
|
start_suite "Proxy waits for weave to be ready before running container commands"
|
||||||
weave_on $HOST1 launch-proxy
|
|
||||||
BASE_IMAGE=busybox
|
BASE_IMAGE=busybox
|
||||||
# Ensure the base image does not exist, so that it will be pulled
|
# Ensure the base image does not exist, so that it will be pulled
|
||||||
! docker_on $HOST1 inspect --format=" " $BASE_IMAGE >/dev/null 2>&1 || docker_on $HOST1 rmi $BASE_IMAGE
|
! docker_on $HOST1 inspect --format=" " $BASE_IMAGE >/dev/null 2>&1 || docker_on $HOST1 rmi $BASE_IMAGE
|
||||||
|
|
||||||
assert_raises "proxy docker_on $HOST1 run --name c1 -e 'WEAVE_CIDR=10.2.1.1/24' $BASE_IMAGE $CHECK_ETHWE_UP"
|
# check that interface is ready, with and without --no-multicast-route
|
||||||
|
weave_on $HOST1 launch-proxy --no-multicast-route
|
||||||
|
check_iface_ready 10.2.1.1/24
|
||||||
|
weave_on $HOST1 stop-proxy
|
||||||
|
weave_on $HOST1 launch-proxy
|
||||||
|
check_iface_ready 10.2.1.1/24
|
||||||
|
|
||||||
# Check committed containers only have one weavewait prepended
|
# Check committed containers only have one weavewait prepended
|
||||||
|
proxy_start_container $HOST1 --name c1 -e 'WEAVE_CIDR=10.2.1.1/24'
|
||||||
COMMITTED_IMAGE=$(proxy docker_on $HOST1 commit c1)
|
COMMITTED_IMAGE=$(proxy docker_on $HOST1 commit c1)
|
||||||
assert_raises "proxy docker_on $HOST1 run --name c2 $COMMITTED_IMAGE"
|
assert_raises "proxy docker_on $HOST1 run --name c2 $COMMITTED_IMAGE"
|
||||||
assert "entrypoint c2" "$(entrypoint $COMMITTED_IMAGE)"
|
assert "entrypoint c2" "$(entrypoint $COMMITTED_IMAGE)"
|
||||||
|
|||||||
Executable
+24
@@ -0,0 +1,24 @@
|
|||||||
|
#! /bin/bash
|
||||||
|
|
||||||
|
. ./config.sh
|
||||||
|
|
||||||
|
C1_NAME=c1.weave.local
|
||||||
|
|
||||||
|
start_suite "Recovery of container details on router restart (via 'local' plugin)"
|
||||||
|
|
||||||
|
weave_on $HOST1 launch
|
||||||
|
start_container_local_plugin $HOST1 --name=c1 --hostname=$C1_NAME
|
||||||
|
c1ip=$(container_ip $HOST1 c1)
|
||||||
|
|
||||||
|
weave_on $HOST1 stop-router
|
||||||
|
weave_on $HOST1 launch-router
|
||||||
|
|
||||||
|
assert "container_ip $HOST1 c1" "$c1ip"
|
||||||
|
assert "weave_on $HOST1 dns-lookup $C1_NAME" "$c1ip"
|
||||||
|
|
||||||
|
# check that c1 IP has been reclaimed and doesn't get assigned to a
|
||||||
|
# fresh container
|
||||||
|
start_container $HOST1 --name=c2
|
||||||
|
assert_raises "container_ip $HOST1 c2 | grep -v $c1ip"
|
||||||
|
|
||||||
|
end_suite
|
||||||
+1
-1
@@ -176,7 +176,7 @@ proxy_start_container_with_dns() {
|
|||||||
rm_containers() {
|
rm_containers() {
|
||||||
host=$1
|
host=$1
|
||||||
shift
|
shift
|
||||||
[ $# -eq 0 ] || docker_on $host rm -f "$@" >/dev/null
|
[ $# -eq 0 ] || docker_on $host rm -f -v "$@" >/dev/null
|
||||||
}
|
}
|
||||||
|
|
||||||
container_ip() {
|
container_ip() {
|
||||||
|
|||||||
@@ -344,6 +344,7 @@ IMAGE=$BASE_IMAGE:$IMAGE_VERSION
|
|||||||
BASE_PLUGIN_IMAGE=$DOCKERHUB_USER/plugin
|
BASE_PLUGIN_IMAGE=$DOCKERHUB_USER/plugin
|
||||||
PLUGIN_IMAGE=$BASE_PLUGIN_IMAGE:$IMAGE_VERSION
|
PLUGIN_IMAGE=$BASE_PLUGIN_IMAGE:$IMAGE_VERSION
|
||||||
PLUGIN_CONTAINER_NAME=weaveplugin
|
PLUGIN_CONTAINER_NAME=weaveplugin
|
||||||
|
VOLUMES_CONTAINER_NAME=weavevolumes-$IMAGE_VERSION
|
||||||
|
|
||||||
PROCFS=${PROCFS:-/proc}
|
PROCFS=${PROCFS:-/proc}
|
||||||
DOCKER_BRIDGE=${DOCKER_BRIDGE:-docker0}
|
DOCKER_BRIDGE=${DOCKER_BRIDGE:-docker0}
|
||||||
@@ -762,11 +763,11 @@ connect_container_to_bridge() {
|
|||||||
# veth pair in the 'up' or 'down' state. Furthermore, moving one
|
# veth pair in the 'up' or 'down' state. Furthermore, moving one
|
||||||
# end into a netns normally sets them to 'down'. But, moving an
|
# end into a netns normally sets them to 'down'. But, moving an
|
||||||
# end into the default netns is a null operation, so they stay up,
|
# end into the default netns is a null operation, so they stay up,
|
||||||
# which leads to errors later on. So to get a consistent result,
|
# which leads to errors in 'set name' since that requires the
|
||||||
# we have to be explicit in setting them 'up' or 'down' as
|
# interface to be down. So to get a consistent result, we set the
|
||||||
# necessary.
|
# interface 'down' to begin with.
|
||||||
if ! netnsenter ip link set $GUEST_IFNAME down ||
|
if ! netnsenter ip link set $GUEST_IFNAME down ||
|
||||||
! netnsenter ip link set $GUEST_IFNAME name $1 up ||
|
! netnsenter ip link set $GUEST_IFNAME name $1 ||
|
||||||
! ip link set $LOCAL_IFNAME up ||
|
! ip link set $LOCAL_IFNAME up ||
|
||||||
! add_iface_$BRIDGE_TYPE $LOCAL_IFNAME ||
|
! add_iface_$BRIDGE_TYPE $LOCAL_IFNAME ||
|
||||||
! configure_arp_cache $1 "netnsenter" ; then
|
! configure_arp_cache $1 "netnsenter" ; then
|
||||||
@@ -833,8 +834,12 @@ setup_router_iface_bridged_fastdp() {
|
|||||||
setup_router_iface_fastdp "$@"
|
setup_router_iface_fastdp "$@"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
container_in_host_ns() {
|
||||||
|
[ -h "$PROCFS/$CONTAINER_PID/ns/net" -a -h "/proc/self/ns/net" -a "$(readlink $PROCFS/$CONTAINER_PID/ns/net)" = "$(readlink /proc/self/ns/net)" ]
|
||||||
|
}
|
||||||
|
|
||||||
attach() {
|
attach() {
|
||||||
if [ -h "$PROCFS/$CONTAINER_PID/ns/net" -a -h "/proc/self/ns/net" -a "$(readlink $PROCFS/$CONTAINER_PID/ns/net)" = "$(readlink /proc/self/ns/net)" ] ; then
|
if container_in_host_ns ; then
|
||||||
echo "Container is running in the host network namespace, and therefore cannot be" >&2
|
echo "Container is running in the host network namespace, and therefore cannot be" >&2
|
||||||
echo "connected to weave. Perhaps the container was started with --net=host." >&2
|
echo "connected to weave. Perhaps the container was started with --net=host." >&2
|
||||||
return 1
|
return 1
|
||||||
@@ -895,8 +900,8 @@ detach() {
|
|||||||
netnsenter ip link del $CONTAINER_IFNAME type veth
|
netnsenter ip link del $CONTAINER_IFNAME type veth
|
||||||
}
|
}
|
||||||
|
|
||||||
container_weave_addrs() {
|
container_ip_addr_show() {
|
||||||
netnsenter ip addr show dev $CONTAINER_IFNAME
|
! container_in_host_ns && netnsenter ip addr show
|
||||||
}
|
}
|
||||||
|
|
||||||
######################################################################
|
######################################################################
|
||||||
@@ -1027,6 +1032,17 @@ wait_for_status() {
|
|||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Parse the output of `ip addr show`, outputing weave addresses of kind $1
|
||||||
|
addrs_from_ip_addr_show() {
|
||||||
|
while read first second rest ; do
|
||||||
|
if [ "$first" != "${first%:}" ] ; then
|
||||||
|
iname="${second%:}"
|
||||||
|
elif [ "$first" = "$1" -a \( "$iname" = "weave" -o "$iname" != "${iname#ethwe}" \) ] ; then
|
||||||
|
echo $second
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
# Call $1 for all containers, passing container ID, all MACs and all IPs
|
# Call $1 for all containers, passing container ID, all MACs and all IPs
|
||||||
with_container_addresses() {
|
with_container_addresses() {
|
||||||
COMMAND=$1
|
COMMAND=$1
|
||||||
@@ -1035,12 +1051,14 @@ with_container_addresses() {
|
|||||||
if [ "$CONTAINER_ID" = "weave:expose" ] ; then
|
if [ "$CONTAINER_ID" = "weave:expose" ] ; then
|
||||||
ADDRS_CMD="ip addr show dev $BRIDGE"
|
ADDRS_CMD="ip addr show dev $BRIDGE"
|
||||||
else
|
else
|
||||||
ADDRS_CMD="with_container_netns $CONTAINER_ID container_weave_addrs"
|
ADDRS_CMD="with_container_netns $CONTAINER_ID container_ip_addr_show"
|
||||||
fi
|
fi
|
||||||
if CONTAINER_ADDRS=$($ADDRS_CMD 2>/dev/null) ; then
|
if CONTAINER_ADDRS=$($ADDRS_CMD 2>/dev/null) ; then
|
||||||
CONTAINER_MAC=$(echo "$CONTAINER_ADDRS" | grep -o 'link/ether .*' | cut -d ' ' -f 2)
|
CONTAINER_MAC=$(echo "$CONTAINER_ADDRS" | addrs_from_ip_addr_show 'link/ether')
|
||||||
CONTAINER_IPS=$(echo "$CONTAINER_ADDRS" | grep -o 'inet .*' | cut -d ' ' -f 2)
|
CONTAINER_IPS=$(echo "$CONTAINER_ADDRS" | addrs_from_ip_addr_show 'inet')
|
||||||
$COMMAND "$CONTAINER_ID" "$CONTAINER_MAC" "$CONTAINER_IPS"
|
if [ -n "$CONTAINER_MAC$CONTAINER_IPS" ] ; then
|
||||||
|
$COMMAND "$CONTAINER_ID" "$CONTAINER_MAC" "$CONTAINER_IPS"
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
@@ -1278,7 +1296,7 @@ collect_dns_add_remove_args() {
|
|||||||
# with_container_addresses.
|
# with_container_addresses.
|
||||||
ipam_reclaim() {
|
ipam_reclaim() {
|
||||||
for CIDR in $3 ; do
|
for CIDR in $3 ; do
|
||||||
# NB: CONTAINER_IP is the IP of the weave container; it is set by wait_for_status.
|
# NB: CONTAINER_IP is the IP of the weave container.
|
||||||
http_call $CONTAINER_IP:$HTTP_PORT PUT /ip/$1/${CIDR%/*}?noErrorOnUnknown=true
|
http_call $CONTAINER_IP:$HTTP_PORT PUT /ip/$1/${CIDR%/*}?noErrorOnUnknown=true
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
@@ -1476,7 +1494,7 @@ proxy_args() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
proxy_addrs() {
|
proxy_addrs() {
|
||||||
if addr="$(http_call_unix $PROXY_CONTAINER_NAME status.sock GET /status 2>&1)" ; then
|
if addr="$(http_call_unix $PROXY_CONTAINER_NAME status.sock GET /status 2>/dev/null)" ; then
|
||||||
echo "$addr" | sed "s/0.0.0.0/$PROXY_HOST/g"
|
echo "$addr" | sed "s/0.0.0.0/$PROXY_HOST/g"
|
||||||
|
|
||||||
else
|
else
|
||||||
@@ -1650,7 +1668,7 @@ attach_router() {
|
|||||||
if [ -z "$NO_DNS_OPT" ] ; then
|
if [ -z "$NO_DNS_OPT" ] ; then
|
||||||
# Tell the newly-started weaveDNS about existing weave IPs
|
# Tell the newly-started weaveDNS about existing weave IPs
|
||||||
for CONTAINER in $(docker ps -q --no-trunc) ; do
|
for CONTAINER in $(docker ps -q --no-trunc) ; do
|
||||||
if CONTAINER_IPS=$(with_container_netns $CONTAINER container_weave_addrs 2>&1 | sed -n -e 's/inet \([^/]*\)\/\(.*\)/\1/p') ; then
|
if CONTAINER_IPS=$(with_container_netns $CONTAINER container_ip_addr_show 2>/dev/null | addrs_from_ip_addr_show 'inet') && [ -n "$CONTAINER_IPS" ] ; then
|
||||||
with_container_fqdn $CONTAINER put_dns_fqdn $CONTAINER_IPS
|
with_container_fqdn $CONTAINER put_dns_fqdn $CONTAINER_IPS
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
@@ -1669,8 +1687,14 @@ launch_proxy() {
|
|||||||
docker_client_args $DOCKER_CLIENT_ARGS
|
docker_client_args $DOCKER_CLIENT_ARGS
|
||||||
proxy_args "$@"
|
proxy_args "$@"
|
||||||
mkdir -p /var/run/weave
|
mkdir -p /var/run/weave
|
||||||
|
# Create a data-only container to mount the weavewait files from
|
||||||
|
if ! docker inspect -f ' ' $VOLUMES_CONTAINER_NAME > /dev/null 2>&1 ; then
|
||||||
|
docker create -v /w -v /w-noop -v /w-nomcast --name=$VOLUMES_CONTAINER_NAME \
|
||||||
|
--label=weavevolumes --entrypoint=/bin/false $EXEC_IMAGE >/dev/null
|
||||||
|
fi
|
||||||
PROXY_CONTAINER=$(docker run --privileged -d --name=$PROXY_CONTAINER_NAME --net=host \
|
PROXY_CONTAINER=$(docker run --privileged -d --name=$PROXY_CONTAINER_NAME --net=host \
|
||||||
$PROXY_VOLUMES \
|
$PROXY_VOLUMES \
|
||||||
|
--volumes-from $VOLUMES_CONTAINER_NAME \
|
||||||
$(docker_sock_options) \
|
$(docker_sock_options) \
|
||||||
-v /var/run/weave:/var/run/weave \
|
-v /var/run/weave:/var/run/weave \
|
||||||
-v /proc:/hostproc \
|
-v /proc:/hostproc \
|
||||||
@@ -1715,8 +1739,12 @@ remove_plugin_network() {
|
|||||||
$PLUGIN_IMAGE --remove-network
|
$PLUGIN_IMAGE --remove-network
|
||||||
}
|
}
|
||||||
|
|
||||||
|
warn_plugin_active() {
|
||||||
|
echo "WARNING: docker may hang in this state; re-launch weave and remove attached containers before shutting down" >&2
|
||||||
|
}
|
||||||
|
|
||||||
stop_plugin() {
|
stop_plugin() {
|
||||||
remove_plugin_network
|
remove_plugin_network || warn_plugin_active
|
||||||
stop $PLUGIN_CONTAINER_NAME "Plugin"
|
stop $PLUGIN_CONTAINER_NAME "Plugin"
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2152,14 +2180,15 @@ EOF
|
|||||||
;;
|
;;
|
||||||
reset)
|
reset)
|
||||||
[ $# -eq 0 ] || usage
|
[ $# -eq 0 ] || usage
|
||||||
plugin_disabled || remove_plugin_network || \
|
plugin_disabled || remove_plugin_network || warn_plugin_active
|
||||||
echo "WARNING: docker may hang in this state; re-launch weave and remove attached containers before shutting down" >&2
|
|
||||||
warn_if_stopping_proxy_in_env
|
warn_if_stopping_proxy_in_env
|
||||||
call_weave DELETE /peer >/dev/null 2>&1 || true
|
call_weave DELETE /peer >/dev/null 2>&1 || true
|
||||||
for NAME in $PLUGIN_CONTAINER_NAME $CONTAINER_NAME $PROXY_CONTAINER_NAME ; do
|
for NAME in $PLUGIN_CONTAINER_NAME $CONTAINER_NAME $PROXY_CONTAINER_NAME ; do
|
||||||
docker stop $NAME >/dev/null 2>&1 || true
|
docker stop $NAME >/dev/null 2>&1 || true
|
||||||
docker rm -f $NAME >/dev/null 2>&1 || true
|
docker rm -f $NAME >/dev/null 2>&1 || true
|
||||||
done
|
done
|
||||||
|
VOLUME_CONTAINERS=$(docker ps -qa --filter label=weavevolumes)
|
||||||
|
[ -n "$VOLUME_CONTAINERS" ] && docker rm -v $VOLUME_CONTAINERS >/dev/null 2>&1 || true
|
||||||
conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true
|
conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true
|
||||||
destroy_bridge
|
destroy_bridge
|
||||||
for LOCAL_IFNAME in $(ip link show | grep v${CONTAINER_IFNAME}pl | cut -d ' ' -f 2 | tr -d ':') ; do
|
for LOCAL_IFNAME in $(ip link show | grep v${CONTAINER_IFNAME}pl | cut -d ' ' -f 2 | tr -d ':') ; do
|
||||||
|
|||||||
Reference in new issue
Block a user