Enable websockets for apps bound to route services

Add new spec for gorouter.route_services.enable_websockets
Default value is true
Unit & Integration tests with a websocket client

Author: Dariquest

jobs/gorouter/spec

@@ -255,6 +255,10 @@ properties:
       Maximum number of attempts on failing requests against route service URLs.
       The minimum value for this setting is 1. This prevents gorouter from getting blocked by indefinite retries.
     default: 3
+  router.route_services.enable_websockets:
+    description: |
+      Enable websocket connections for application routes bound to Route Services.
+    default: true
   router.route_services.cert_chain:
     description: Certificate chain used for client authentication to TLS-registered route services.  In PEM format.
   router.route_services.private_key:

jobs/gorouter/templates/gorouter.yml.erb

@@ -345,12 +345,14 @@ if (route_service_attempts < 1 )
 end
 
 strict_signature_validation = p('router.route_services_strict_signature_validation')
+enable_websockets = p('router.route_services.enable_websockets')
 
 route_services = {
   'max_attempts' => route_service_attempts,
   'cert_chain' => route_services_cert_chain,
   'private_key' => route_services_private_key,
   'strict_signature_validation' => strict_signature_validation,
+  'enable_websockets' => enable_websockets,
 }
 
 params['route_services'] = route_services

spec/gorouter_templates_spec.rb

@@ -228,7 +228,8 @@
             'max_attempts' => 3,
             'cert_chain' => ROUTE_SERVICES_CLIENT_TEST_CERT,
             'private_key' => ROUTE_SERVICES_CLIENT_TEST_KEY,
-            'strict_signature_validation' => false
+            'strict_signature_validation' => false,
+            'enable_websockets' => true,
           },
           'frontend_idle_timeout' => 5,
           'ip_local_port_range' => '1024 65535',
@@ -919,6 +920,27 @@
             expect(parsed_yaml['route_services']['strict_signature_validation']).to eq(true)
           end
         end
+        context 'when enable_websockets not set' do
+          it 'defaults to true' do
+            expect(parsed_yaml['route_services']['enable_websockets']).to eq(true)
+          end
+        end
+        context 'when enable_websockets is enabled' do
+          before do
+            deployment_manifest_fragment['router']['route_services']['enable_websockets'] = true
+          end
+          it 'parses to true' do
+            expect(parsed_yaml['route_services']['enable_websockets']).to eq(true)
+          end
+        end
+        context 'when enable_websockets is disabled' do
+              before do
+                deployment_manifest_fragment['router']['route_services']['enable_websockets'] = false
+              end
+              it 'parses to true' do
+                expect(parsed_yaml['route_services']['enable_websockets']).to eq(false)
+              end
+            end
       end
 
       describe 'backends' do

src/code.cloudfoundry.org/gorouter/cmd/gorouter/main.go

@@ -175,6 +175,7 @@ func main() {
 		cryptoPrev,
 		c.RouteServiceRecommendHttps,
 		c.RouteServiceConfig.StrictSignatureValidation,
+		c.RouteServiceConfig.EnableWebsockets,
 	)
 
 	// These TLS configs are just templates. If you add other keys you will

src/code.cloudfoundry.org/gorouter/config/config.go

@@ -194,6 +194,7 @@ type RouteServiceConfig struct {
 	MaxAttempts               int              `yaml:"max_attempts"`
 	StrictSignatureValidation bool             `yaml:"strict_signature_validation"`
 	TLSPem                    `yaml:",inline"` // embed to get cert_chain and private_key for client authentication
+	EnableWebsockets          bool             `yaml:"enable_websockets"`
 }
 
 type LoggingConfig struct {

src/code.cloudfoundry.org/gorouter/handlers/routeservice.go

@@ -104,8 +104,7 @@ func (r *RouteService) ServeHTTP(rw http.ResponseWriter, req *http.Request, next
 		)
 		return
 	}
-
-	if IsWebSocketUpgrade(req) {
+	if IsWebSocketUpgrade(req) && !r.config.EnableWebsockets() {
 		logger.Info("route-service-unsupported")
 		AddRouterErrorHeader(rw, "route_service_unsupported")
 		r.errorWriter.WriteError(

src/code.cloudfoundry.org/gorouter/handlers/routeservice_test.go

@@ -99,8 +99,7 @@ var _ = Describe("Route Service Handler", func() {
 		crypto, err = secure.NewAesGCM([]byte("ABCDEFGHIJKLMNOP"))
 		Expect(err).NotTo(HaveOccurred())
 		config = routeservice.NewRouteServiceConfig(
-			logger.Logger, true, true, nil, 60*time.Second, crypto, nil, true, false,
-		)
+			logger.Logger, true, true, nil, 60*time.Second, crypto, nil, true, false, true)
 
 		nextCalled = false
 		prevHandler = &PrevHandler{}
@@ -121,7 +120,7 @@ var _ = Describe("Route Service Handler", func() {
 
 	Context("with route services disabled", func() {
 		BeforeEach(func() {
-			config = routeservice.NewRouteServiceConfig(logger.Logger, false, false, nil, 0, nil, nil, false, false)
+			config = routeservice.NewRouteServiceConfig(logger.Logger, false, false, nil, 0, nil, nil, false, false, true)
 		})
 
 		Context("for normal routes", func() {
@@ -192,7 +191,7 @@ var _ = Describe("Route Service Handler", func() {
 			Context("with strictSignatureValidation enabled", func() {
 				BeforeEach(func() {
 					config = routeservice.NewRouteServiceConfig(
-						logger.Logger, true, false, nil, 60*time.Second, crypto, nil, false, true,
+						logger.Logger, true, false, nil, 60*time.Second, crypto, nil, false, true, true,
 					)
 				})
 
@@ -274,7 +273,7 @@ var _ = Describe("Route Service Handler", func() {
 					BeforeEach(func() {
 						hairpinning := false
 						config = routeservice.NewRouteServiceConfig(
-							logger.Logger, true, hairpinning, nil, 60*time.Second, crypto, nil, true, false,
+							logger.Logger, true, hairpinning, nil, 60*time.Second, crypto, nil, true, false, true,
 						)
 					})
 
@@ -305,7 +304,7 @@ var _ = Describe("Route Service Handler", func() {
 					BeforeEach(func() {
 						hairpinning := true
 						config = routeservice.NewRouteServiceConfig(
-							logger.Logger, true, hairpinning, nil, 60*time.Second, crypto, nil, true, false,
+							logger.Logger, true, hairpinning, nil, 60*time.Second, crypto, nil, true, false, true,
 						)
 					})
 
@@ -336,7 +335,7 @@ var _ = Describe("Route Service Handler", func() {
 					BeforeEach(func() {
 						hairpinning := true
 						config = routeservice.NewRouteServiceConfig(
-							logger.Logger, true, hairpinning, []string{"route-service.com"}, 60*time.Second, crypto, nil, true, false,
+							logger.Logger, true, hairpinning, []string{"route-service.com"}, 60*time.Second, crypto, nil, true, false, true,
 						)
 					})
 
@@ -368,7 +367,7 @@ var _ = Describe("Route Service Handler", func() {
 					BeforeEach(func() {
 						hairpinning := true
 						config = routeservice.NewRouteServiceConfig(
-							logger.Logger, true, hairpinning, []string{"example.com"}, 60*time.Second, crypto, nil, true, false,
+							logger.Logger, true, hairpinning, []string{"example.com"}, 60*time.Second, crypto, nil, true, false, true,
 						)
 					})
 
@@ -400,7 +399,7 @@ var _ = Describe("Route Service Handler", func() {
 					BeforeEach(func() {
 						hairpinning := true
 						config = routeservice.NewRouteServiceConfig(
-							logger.Logger, true, hairpinning, generateHugeAllowlist(1000000), 60*time.Second, crypto, nil, true, false,
+							logger.Logger, true, hairpinning, generateHugeAllowlist(1000000), 60*time.Second, crypto, nil, true, false, true,
 						)
 					})
 
@@ -438,7 +437,7 @@ var _ = Describe("Route Service Handler", func() {
 			Context("when recommendHttps is set to false", func() {
 				BeforeEach(func() {
 					config = routeservice.NewRouteServiceConfig(
-						logger.Logger, true, false, nil, 60*time.Second, crypto, nil, false, false,
+						logger.Logger, true, false, nil, 60*time.Second, crypto, nil, false, false, true,
 					)
 				})
 				It("sends the request to the route service with X-CF-Forwarded-Url using http scheme", func() {
@@ -609,7 +608,7 @@ var _ = Describe("Route Service Handler", func() {
 					cryptoPrev, err = secure.NewAesGCM([]byte("QRSTUVWXYZ123456"))
 					Expect(err).ToNot(HaveOccurred())
 					config = routeservice.NewRouteServiceConfig(
-						logger.Logger, true, false, nil, 60*time.Second, crypto, cryptoPrev, true, false,
+						logger.Logger, true, false, nil, 60*time.Second, crypto, cryptoPrev, true, false, true,
 					)
 				})
 
@@ -704,13 +703,17 @@ var _ = Describe("Route Service Handler", func() {
 				req.Header.Set("upgrade", "websocket")
 
 			})
-			It("returns a 503", func() {
+			It("request contains correct route service URL", func() {
 				handler.ServeHTTP(resp, req)
 
-				Expect(resp.Code).To(Equal(http.StatusServiceUnavailable))
-				Expect(resp.Body.String()).To(ContainSubstring("Websocket requests are not supported for routes bound to Route Services."))
+				var passedReq *http.Request
+				Eventually(reqChan).Should(Receive(&passedReq))
 
-				Expect(nextCalled).To(BeFalse())
+				reqInfo, err := handlers.ContextRequestInfo(passedReq)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(reqInfo.RouteServiceURL.Scheme).To(Equal("https"))
+				Expect(reqInfo.RouteServiceURL.Host).To(Equal("goodrouteservice.com"))
+				Expect(nextCalled).To(BeTrue(), "Expected the next handler to be called.")
 			})
 		})
 
@@ -888,7 +891,7 @@ var _ = Describe("Route Service Handler", func() {
 				By(testCase.name)
 
 				config = routeservice.NewRouteServiceConfig(
-					logger.Logger, true, true, testCase.allowlist, 60*time.Second, crypto, nil, true, false,
+					logger.Logger, true, true, testCase.allowlist, 60*time.Second, crypto, nil, true, false, true,
 				)
 
 				if testCase.err {

src/code.cloudfoundry.org/gorouter/integration/common_integration_test.go

@@ -83,6 +83,8 @@ func NewTestState() *testState {
 	cfg.SuspendPruningIfNatsUnavailable = true
 
 	cfg.DisableKeepAlives = false
+	cfg.RouteServiceEnabled = true
+	cfg.RouteServiceConfig.EnableWebsockets = true
 
 	externalRouteServiceHostname := "external-route-service.localhost.routing.cf-app.com"
 	routeServiceKey, routeServiceCert := test_util.CreateKeyPair(externalRouteServiceHostname)

src/code.cloudfoundry.org/gorouter/integration/route_services_test.go

@@ -4,29 +4,52 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io"
+	"log"
+	"net"
 	"net/http"
 	"net/http/httptest"
+	"net/http/httputil"
+	"net/url"
+	"os"
 	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"golang.org/x/net/websocket"
 
 	"code.cloudfoundry.org/gorouter/route"
 	"code.cloudfoundry.org/gorouter/test/common"
 )
 
+func wsClient(conn net.Conn, urlStr string) (*websocket.Conn, error) {
+	wsUrl, err := url.ParseRequestURI(urlStr)
+	Expect(err).NotTo(HaveOccurred())
+
+	cfg := &websocket.Config{
+		Location: wsUrl,
+		Origin:   wsUrl,
+		Version:  websocket.ProtocolVersionHybi13,
+	}
+
+	wsConn, err := websocket.NewClient(cfg, conn)
+	return wsConn, err
+}
+
 var _ = Describe("Route services", func() {
 
 	var testState *testState
 
 	const (
-		appHostname = "app-with-route-service.some.domain"
+		appHostname   = "app-with-route-service.some.domain"
+		wsAppHostname = "ws-app-with-route-service.some.domain"
 	)
 
 	var (
-		testApp      *httptest.Server
-		routeService *httptest.Server
+		testApp        *httptest.Server
+		routeService   *httptest.Server
+		wsTestApp      *httptest.Server
+		wsRouteService *httptest.Server
 	)
 
 	BeforeEach(func() {
@@ -69,6 +92,30 @@ var _ = Describe("Route services", func() {
 					Expect(err).ToNot(HaveOccurred())
 				}))
 
+		wsRouteService = httptest.NewUnstartedServer(
+			&httputil.ReverseProxy{
+				Director: func(req *http.Request) {
+					forwardedURLStr := req.Header.Get("X-Cf-Forwarded-Url")
+
+					forwardedURL, err := url.Parse(forwardedURLStr)
+					if err != nil {
+						log.Printf("ERROR: X-Cf-Forwarded-Url unparseable: %s\n", err.Error())
+						return
+					}
+
+					req.URL = &url.URL{
+						Scheme: "http",
+						Host:   fmt.Sprintf("127.0.0.1:%d", testState.cfg.Port),
+					}
+					req.Host = forwardedURL.Host
+				},
+				Transport: &http.Transport{
+					TLSClientConfig: &tls.Config{
+						MinVersion:         tls.VersionTLS12,
+						InsecureSkipVerify: true,
+					},
+				},
+			})
 	})
 
 	AfterEach(func() {
@@ -84,6 +131,57 @@ var _ = Describe("Route services", func() {
 		return fmt.Sprintf("https://%s:%s", testState.trustedExternalServiceHostname, port)
 	}
 
+	Context("Happy Path with a web socket app with a route service", func() {
+		Context("When an app is registered with a simple route service", func() {
+			BeforeEach(func() {
+				testState.EnableAccessLog()
+				testState.StartGorouterOrFail()
+				wsRouteService.Start()
+				nilHandshake := func(c *websocket.Config, request *http.Request) error { return nil }
+				wsHandler := websocket.Server{Handler: func(conn *websocket.Conn) {
+					msgBuf := make([]byte, 100)
+					n, err := conn.Read(msgBuf)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(string(msgBuf[:n])).To(Equal("HELLO WEBSOCKET"))
+
+					_, _ = conn.Write([]byte("WEBSOCKET OK"))
+					conn.Close()
+				}, Handshake: nilHandshake}
+
+				wsTestApp = httptest.NewServer(wsHandler)
+
+				testState.registerWithInternalRouteService(
+					wsTestApp,
+					wsRouteService,
+					wsAppHostname,
+					testState.cfg.SSLPort,
+				)
+			})
+
+			It("succeeds", func() {
+				conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", testState.cfg.Port))
+				Expect(err).NotTo(HaveOccurred())
+
+				wsConn, err := wsClient(conn, "ws://"+wsAppHostname)
+				Expect(err).NotTo(HaveOccurred())
+
+				num, err := wsConn.Write([]byte("HELLO WEBSOCKET"))
+				Expect(err).NotTo(HaveOccurred())
+				Expect(num).To(Equal(len([]byte("HELLO WEBSOCKET"))))
+
+				msgBuf := make([]byte, 100)
+				num2, err := wsConn.Read(msgBuf)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(string(msgBuf[:num2])).To(Equal("WEBSOCKET OK"))
+
+				Eventually(func() ([]byte, error) {
+					return os.ReadFile(testState.AccessLogFilePath())
+				}).Should(ContainSubstring(`"GET / HTTP/1.1" 101 0 0`))
+			})
+		})
+	})
+
 	Context("Happy Path", func() {
 		Context("When an app is registered with a simple route service", func() {
 			BeforeEach(func() {
@@ -102,7 +200,6 @@ var _ = Describe("Route services", func() {
 				req := testState.newGetRequest(
 					fmt.Sprintf("https://%s", appHostname),
 				)
-
 				res, err := testState.client.Do(req)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(res.StatusCode).To(Equal(http.StatusOK))

src/code.cloudfoundry.org/gorouter/proxy/proxy_suite_test.go

@@ -121,6 +121,7 @@ var _ = JustBeforeEach(func() {
 		cryptoPrev,
 		recommendHTTPS,
 		strictSignatureValidation,
+		conf.RouteServiceConfig.EnableWebsockets,
 	)
 
 	proxyServer, err = net.Listen("tcp", "127.0.0.1:0")

src/code.cloudfoundry.org/gorouter/proxy/proxy_unit_test.go

@@ -64,6 +64,7 @@ var _ = Describe("Proxy Unit tests", func() {
 				cryptoPrev,
 				false,
 				false,
+				conf.RouteServiceConfig.EnableWebsockets,
 			)
 			varz := test_helpers.NullVarz{}
 			sender := new(fakes.MetricSender)

src/code.cloudfoundry.org/gorouter/proxy/route_service_test.go

@@ -83,6 +83,7 @@ var _ = Describe("Route Services", func() {
 			nil,
 			recommendHTTPS,
 			strictSignatureValidation,
+			conf.RouteServiceConfig.EnableWebsockets,
 		)
 		reqArgs, err := config.CreateRequest("", forwardedUrl)
 		Expect(err).ToNot(HaveOccurred())