From f816be6f697206958b1d80235d04025e966cc48a Mon Sep 17 00:00:00 2001 From: Jonathan Leibiusky Date: Fri, 17 Feb 2017 11:10:01 -0300 Subject: [PATCH] Add DNS support for PWD instances (#94) * Add DNS support for PWD instances * Store IP address of PWD in all session networks and restore it with the same IP address * Remove unnecesary print * Change url format to pwd-port for better DNS filtering * Make PWD listen on 80 and 443 for DNS resolve to work --- Dockerfile.dind | 5 ++-- README.md | 2 +- api.go | 48 +++++++++++++++++++++++++++--- docker-compose.yml | 4 +-- handlers/reverseproxy.go | 8 ++--- handlers/websocket_reverseproxy.go | 4 +-- services/docker.go | 34 +++++++++++++++++---- services/session.go | 26 +++++++++------- www/assets/app.js | 2 +- 9 files changed, 102 insertions(+), 31 deletions(-) diff --git a/Dockerfile.dind b/Dockerfile.dind index 0b30c0d..9332f95 100644 --- a/Dockerfile.dind +++ b/Dockerfile.dind @@ -2,8 +2,9 @@ FROM docker:1.13.1-dind RUN apk add --no-cache git tmux py2-pip apache2-utils vim build-base gettext-dev curl bash +ENV COMPOSE_VERSION=1.11.1 # Install Compose and Machine -RUN pip install docker-compose==1.10.0-rc1 +RUN pip install docker-compose==${COMPOSE_VERSION} RUN curl -L https://github.com/docker/machine/releases/download/v0.9.0-rc1/docker-machine-Linux-x86_64 \ -o /usr/bin/docker-machine && chmod +x /usr/bin/docker-machine @@ -34,6 +35,6 @@ WORKDIR /root CMD cat /etc/hosts >/etc/hosts.bak && \ sed 's/^::1.*//' /etc/hosts.bak > /etc/hosts && \ dockerd --experimental -g /graph --host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:2375 \ - --storage-driver=$DOCKER_STORAGE_DRIVER &>/docker.log & \ + --storage-driver=$DOCKER_STORAGE_DRIVER --dns $PWD_IP_ADDRESS --dns 8.8.8.8 &>/docker.log & \ while true ; do /bin/bash ; done # ... and then put a shell in the foreground, restarting it if it exits diff --git a/README.md b/README.md index f9d4a09..680aa88 100644 --- a/README.md +++ b/README.md @@ -52,5 +52,5 @@ Notes: ~~We're planning to setup a reverse proxy that handles redirection automatically, in the meantime you can use [ngrok](https://ngrok.com) within PWD running `docker run --name supergrok -d jpetazzo/supergrok` then `docker logs --follow supergrok` , it will give you a ngrok URL, now you can go to that URL and add the IP+port that you want to connect to… e.g. if your PWD instance is 10.0.42.3, you can go to http://xxxxxx.ngrok.io/10.0.42.3:8000 (where the xxxxxx is given to you in the supergrok logs).~~ -If you need to access your services from outside, use the following URL pattern `http://ip-.play-with-docker.com` (i.e: http://ip10_2_135_3-80.play-with-docker.com/). +If you need to access your services from outside, use the following URL pattern `http://pwd-.play-with-docker.com` (i.e: http://pwd10_2_135_3-80.play-with-docker.com/). diff --git a/api.go b/api.go index e52ec25..4358d9c 100644 --- a/api.go +++ b/api.go @@ -6,6 +6,7 @@ import ( "log" "net/http" "os" + "regexp" "strings" "github.com/franela/play-with-docker/config" @@ -14,6 +15,7 @@ import ( "github.com/franela/play-with-docker/templates" gh "github.com/gorilla/handlers" "github.com/gorilla/mux" + "github.com/miekg/dns" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/urfave/negroni" "github.com/yhat/wsutil" @@ -25,6 +27,16 @@ func main() { bypassCaptcha := len(os.Getenv("GOOGLE_RECAPTCHA_DISABLED")) > 0 + // Start the DNS server + dnsServer := &dns.Server{Addr: ":53", Net: "udp"} + dns.HandleFunc(".", handleDnsRequest) + go func() { + err := dnsServer.ListenAndServe() + if err != nil { + log.Fatal(err) + } + }() + server := services.CreateWSServer() server.On("connection", handlers.WS) server.On("error", handlers.WSError) @@ -49,8 +61,8 @@ func main() { } // Specific routes - r.Host(`{node:ip[0-9]{1,3}_[0-9]{1,3}_[0-9]{1,3}_[0-9]{1,3}}-{port:[0-9]*}.{tld:.*}`).HandlerFunc(proxyMultiplexer) - r.Host(`{node:ip[0-9]{1,3}_[0-9]{1,3}_[0-9]{1,3}_[0-9]{1,3}}.{tld:.*}`).HandlerFunc(proxyMultiplexer) + r.Host(`{node:pwd[0-9]{1,3}_[0-9]{1,3}_[0-9]{1,3}_[0-9]{1,3}}-{port:[0-9]*}.{tld:.*}`).HandlerFunc(proxyMultiplexer) + r.Host(`{node:pwd[0-9]{1,3}_[0-9]{1,3}_[0-9]{1,3}_[0-9]{1,3}}.{tld:.*}`).HandlerFunc(proxyMultiplexer) r.HandleFunc("/ping", handlers.Ping).Methods("GET") r.HandleFunc("/sessions/{sessionId}", handlers.GetSession).Methods("GET") r.Handle("/sessions/{sessionId}/instances", http.HandlerFunc(handlers.NewInstance)).Methods("POST") @@ -98,7 +110,7 @@ func main() { ssl := mux.NewRouter() sslProxyHandler := handlers.NewSSLDaemonHandler() - ssl.Host(`{node:ip[0-9]{1,3}_[0-9]{1,3}_[0-9]{1,3}_[0-9]{1,3}}-2375.{tld:.*}`).Handler(sslProxyHandler) + ssl.Host(`{node:pwd[0-9]{1,3}_[0-9]{1,3}_[0-9]{1,3}_[0-9]{1,3}}-2375.{tld:.*}`).Handler(sslProxyHandler) log.Println("Listening TLS on port " + config.SSLPortNumber) s := &http.Server{Addr: "0.0.0.0:" + config.SSLPortNumber, Handler: ssl} @@ -107,7 +119,7 @@ func main() { chunks := strings.Split(clientHello.ServerName, ".") chunks = strings.Split(chunks[0], "-") - ip := strings.Replace(strings.TrimPrefix(chunks[0], "ip"), "_", ".", -1) + ip := strings.Replace(strings.TrimPrefix(chunks[0], "pwd"), "_", ".", -1) i := services.FindInstanceByIP(ip) if i == nil { return nil, fmt.Errorf("Instance %s doesn't exist", clientHello.ServerName) @@ -119,3 +131,31 @@ func main() { } log.Fatal(s.ListenAndServeTLS("", "")) } + +var dnsFilter = regexp.MustCompile(`pwd[0-9]{1,3}_[0-9]{1,3}_[0-9]{1,3}_[0-9]{1,3}`) + +func handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) { + if len(r.Question) > 0 && dnsFilter.MatchString(r.Question[0].Name) { + // this is something we know about and we should try to handle + question := r.Question[0].Name + domainChunks := strings.Split(question, ".") + tldChunks := strings.Split(strings.TrimPrefix(domainChunks[0], "pwd"), "-") + ip := strings.Replace(tldChunks[0], "_", ".", -1) + + m := new(dns.Msg) + m.SetReply(r) + m.Authoritative = true + m.RecursionAvailable = true + a, err := dns.NewRR(fmt.Sprintf("%s 60 IN A %s", question, ip)) + if err != nil { + log.Fatal(err) + } + m.Answer = append(m.Answer, a) + w.WriteMsg(m) + return + } else { + // we have no information about this and we are not a recursive dns server, so we just fail so the client can fallback to the next dns server it has configured + dns.HandleFailed(w, r) + return + } +} diff --git a/docker-compose.yml b/docker-compose.yml index a29941a..706bc1e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,8 +9,8 @@ services: command: /bin/sh -c 'cd /go/src/github.com/franela/play-with-docker; go run api.go' ports: # app exposes port 3000 - - "3000:3000" - - "3001:3001" + - "80:3000" + - "443:3001" volumes: # since this app creates networks and launches containers, we need to talk to docker daemon - /var/run/docker.sock:/var/run/docker.sock diff --git a/handlers/reverseproxy.go b/handlers/reverseproxy.go index d9860fd..49f6520 100644 --- a/handlers/reverseproxy.go +++ b/handlers/reverseproxy.go @@ -40,9 +40,9 @@ func NewMultipleHostReverseProxy() *httputil.ReverseProxy { port = "80" } - if strings.HasPrefix(node, "ip") { + if strings.HasPrefix(node, "pwd") { // Node is actually an ip, need to convert underscores by dots. - ip := strings.Replace(strings.TrimPrefix(node, "ip"), "_", ".", -1) + ip := strings.Replace(strings.TrimPrefix(node, "pwd"), "_", ".", -1) if net.ParseIP(ip) == nil { // Not a valid IP, so treat this is a hostname. @@ -69,9 +69,9 @@ func NewSSLDaemonHandler() *httputil.ReverseProxy { director := func(req *http.Request) { v := mux.Vars(req) node := v["node"] - if strings.HasPrefix(node, "ip") { + if strings.HasPrefix(node, "pwd") { // Node is actually an ip, need to convert underscores by dots. - ip := strings.Replace(strings.TrimPrefix(node, "ip"), "_", ".", -1) + ip := strings.Replace(strings.TrimPrefix(node, "pwd"), "_", ".", -1) if net.ParseIP(ip) == nil { // Not a valid IP, so treat this is a hostname. diff --git a/handlers/websocket_reverseproxy.go b/handlers/websocket_reverseproxy.go index c91c25e..e4bd17e 100644 --- a/handlers/websocket_reverseproxy.go +++ b/handlers/websocket_reverseproxy.go @@ -29,9 +29,9 @@ func NewMultipleHostWebsocketReverseProxy() *wsutil.ReverseProxy { port = "80" } - if strings.HasPrefix(node, "ip") { + if strings.HasPrefix(node, "pwd") { // Node is actually an ip, need to convert underscores by dots. - ip := strings.Replace(strings.TrimPrefix(node, "ip"), "_", ".", -1) + ip := strings.Replace(strings.TrimPrefix(node, "pwd"), "_", ".", -1) if net.ParseIP(ip) == nil { // Not a valid IP, so treat this is a hostname. diff --git a/services/docker.go b/services/docker.go index 6b99dc8..29d334a 100644 --- a/services/docker.go +++ b/services/docker.go @@ -135,17 +135,33 @@ func CreateNetwork(name string) error { return nil } -func ConnectNetwork(containerId, networkId string) error { - err := c.NetworkConnect(context.Background(), networkId, containerId, &network.EndpointSettings{}) +func ConnectNetwork(containerId, networkId, ip string) (string, error) { + settings := &network.EndpointSettings{} + if ip != "" { + settings.IPAddress = ip + } + err := c.NetworkConnect(context.Background(), networkId, containerId, settings) if err != nil && !strings.Contains(err.Error(), "already exists") { log.Printf("Connection container to network err [%s]\n", err) - return err + return "", err } - return nil + // Obtain the IP of the PWD container in this network + container, err := c.ContainerInspect(context.Background(), containerId) + if err != nil { + return "", err + } + + n, found := container.NetworkSettings.Networks[networkId] + if !found { + return "", fmt.Errorf("Container [%s] connected to the network [%s] but couldn't obtain it's IP address", containerId, networkId) + } + + return n.IPAddress, nil } + func DisconnectNetwork(containerId, networkId string) error { err := c.NetworkDisconnect(context.Background(), networkId, containerId, true) @@ -217,7 +233,15 @@ func CreateInstance(session *Session, dindImage string) (*Instance, error) { break } } - conf := &container.Config{Hostname: nodeName, Image: dindImage, Tty: true, OpenStdin: true, AttachStdin: true, AttachStdout: true, AttachStderr: true} + conf := &container.Config{Hostname: nodeName, + Image: dindImage, + Tty: true, + OpenStdin: true, + AttachStdin: true, + AttachStdout: true, + AttachStderr: true, + Env: []string{fmt.Sprintf("PWD_IP_ADDRESS=%s", session.PwdIpAddress)}, + } networkConf := &network.NetworkingConfig{ map[string]*network.EndpointSettings{ session.Id: &network.EndpointSettings{Aliases: []string{nodeName}}, diff --git a/services/session.go b/services/session.go index 8983106..adbf151 100644 --- a/services/session.go +++ b/services/session.go @@ -42,14 +42,15 @@ func init() { var wsServer *socketio.Server type Session struct { - rw sync.Mutex - Id string `json:"id"` - Instances map[string]*Instance `json:"instances"` - clients []*Client `json:"-"` - CreatedAt time.Time `json:"created_at"` - ExpiresAt time.Time `json:"expires_at"` - scheduled bool `json:"-"` - ticker *time.Ticker `json:"-"` + rw sync.Mutex + Id string `json:"id"` + Instances map[string]*Instance `json:"instances"` + clients []*Client `json:"-"` + CreatedAt time.Time `json:"created_at"` + ExpiresAt time.Time `json:"expires_at"` + scheduled bool `json:"-"` + ticker *time.Ticker `json:"-"` + PwdIpAddress string `json:"pwd_ip_address"` } func (s *Session) Lock() { @@ -239,10 +240,12 @@ func NewSession(duration time.Duration) (*Session, error) { log.Printf("Network [%s] created for session [%s]\n", s.Id, s.Id) // Connect PWD daemon to the new network - if err := ConnectNetwork("pwd", s.Id); err != nil { + ip, err := ConnectNetwork("pwd", s.Id, "") + if err != nil { log.Println("ERROR NETWORKING") return nil, err } + s.PwdIpAddress = ip log.Printf("Connected pwd to network [%s]\n", s.Id) // Schedule peridic tasks execution @@ -315,7 +318,10 @@ func LoadSessionsFromDisk() error { } // Connect PWD daemon to the new network - if err := ConnectNetwork("pwd", s.Id); err != nil { + if s.PwdIpAddress == "" { + log.Fatal("Cannot load stored sessions as they don't have the pwd ip address stored with them") + } + if _, err := ConnectNetwork("pwd", s.Id, s.PwdIpAddress); err != nil { if strings.Contains(err.Error(), "Could not attach to network") { log.Printf("Network for session [%s] doesn't exist. Removing all instances and session.", s.Id) CloseSession(s) diff --git a/www/assets/app.js b/www/assets/app.js index 94d2c52..c4d939f 100644 --- a/www/assets/app.js +++ b/www/assets/app.js @@ -176,7 +176,7 @@ } $scope.getProxyUrl = function(instance, port) { - var url = window.location.protocol + '//ip' + instance.ip.replace(/\./g, '_') + '-' + port + '.' + window.location.host; + var url = window.location.protocol + '//pwd' + instance.ip.replace(/\./g, '_') + '-' + port + '.' + window.location.host; return url; }