Introduce an ACL for blocking on tcp layer

acceptance-tests/access_control_test.go

@@ -11,10 +11,10 @@ import (
 )
 
 /*
-	Test strategy:
-		* Use an SSH tunnel to make requests to HAProxy that appear to come from 127.0.0.1
-		* Requests directly from test runner on Concourse appear to come from 10.0.0.0/8
-		We can test whitelisting and blacklisting by using these CIDRs
+Test strategy:
+  - Use an SSH tunnel to make requests to HAProxy that appear to come from 127.0.0.1
+  - Requests directly from test runner on Concourse appear to come from 10.0.0.0/8
+    We can test whitelisting and blacklisting by using these CIDRs
 */
 var _ = Describe("Access Control", func() {
 	opsfileWhitelist := `---
@@ -32,6 +32,12 @@ var _ = Describe("Access Control", func() {
   path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/cidr_blacklist?
   value: ((cidr_blacklist))
 `
+	opsfileTCPBlacklist := `---
+# Enable TCP-layer CIDR blacklist
+- type: replace
+  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/tcp_blacklist_cidrs?
+  value: ((tcp_blacklist_cidrs))
+`
 
 	It("Allows IPs in whitelisted CIDRS", func() {
 		haproxyBackendPort := 12000
@@ -97,4 +103,36 @@ var _ = Describe("Access Control", func() {
 		By("Allowing access to non-blacklisted CIDRs (request from 127.0.0.1 on HAProxy VM)")
 		expectTestServer200(http.Get("http://127.0.0.1:11000"))
 	})
+
+	It("Rejects IPs in TCP-layer blocklisted CIDRs", func() {
+		haproxyBackendPort := 12000
+
+		haproxyInfo, _ := deployHAProxy(baseManifestVars{
+			haproxyBackendPort:    haproxyBackendPort,
+			haproxyBackendServers: []string{"127.0.0.1"},
+			deploymentName:        deploymentNameForTestNode(),
+		}, []string{opsfileTCPBlacklist}, map[string]interface{}{
+			// traffic from test runner appears to come from this CIDR block
+			"tcp_blacklist_cidrs": []string{"10.0.0.0/8"},
+		}, true)
+
+		closeLocalServer, localPort := startDefaultTestServer()
+		defer closeLocalServer()
+
+		closeBackendTunnel := setupTunnelFromHaproxyToTestServer(haproxyInfo, haproxyBackendPort, localPort)
+		defer closeBackendTunnel()
+
+		// Set up a tunnel so that requests to localhost:11000 appear to come from 127.0.0.1
+		// on the HAProxy VM, which is NOT in the TCP blocklist
+		closeFrontendTunnel := setupTunnelFromLocalMachineToHAProxy(haproxyInfo, 11000, 80)
+		defer closeFrontendTunnel()
+
+		By("Denying TCP connections from blocklisted CIDRs (request from test runner)")
+		_, err := http.Get(fmt.Sprintf("http://%s", haproxyInfo.PublicIP))
+		Expect(err).To(HaveOccurred())
+		Expect(errors.Is(err, io.EOF)).To(BeTrue())
+
+		By("Allowing TCP connections from non-blocklisted CIDRs (request from 127.0.0.1 on HAProxy VM)")
+		expectTestServer200(http.Get("http://127.0.0.1:11000"))
+	})
 })

jobs/haproxy/spec

@@ -22,6 +22,7 @@ templates:
   backend-crt.erb:              config/backend-crt.pem
   client-revocation-list.erb:   config/client-revocation-list.pem
   blacklist_cidrs.txt.erb:      config/blacklist_cidrs.txt
+  tcp_blacklist_cidrs.txt.erb:  config/tcp_blacklist_cidrs.txt
   whitelist_cidrs.txt.erb:      config/whitelist_cidrs.txt
   expect_proxy_cidrs.txt.erb:   config/expect_proxy_cidrs.txt
   trusted_domain_cidrs.txt.erb: config/trusted_domain_cidrs.txt
@@ -593,6 +594,13 @@ properties:
       cidr_blacklist:
       - 10.0.0.0/8
       - 192.168.2.0/24
+  ha_proxy.tcp_blacklist_cidrs:
+    description: List of CIDRs to block on TCP level. If empty, only a comment is rendered. Format is string array of CIDRs or single string of base64 encoded gzip.
+    default: ~
+    example:
+      tcp_blacklist_cidrs:
+        - 10.0.0.0/8
+        - 192.168.2.0/24
   ha_proxy.cidr_whitelist:
     description: "List of CIDRs to allow for http(s). Format is string array of CIDRs or single string of base64 encoded gzip. Note that unless ha_proxy.block_all is true, non-whitelisted traffic will still be allowed, provided that traffic is not also blacklisted"
     default: ~

jobs/haproxy/templates/haproxy.config.erb

@@ -428,6 +428,8 @@ frontend http-in
   <%- if properties.ha_proxy.frontend_config -%>
     <%= format_indented_multiline_config(p("ha_proxy.frontend_config")) %>
   <%- end -%>
+    acl layer4_block src -f /var/vcap/jobs/haproxy/config/tcp_blacklist_cidrs.txt
+    tcp-request connection reject if layer4_block
   <%- if_p("ha_proxy.connections_rate_limit.table_size", "ha_proxy.connections_rate_limit.window_size") do -%>
     tcp-request connection track-sc0 src table st_tcp_conn_rate
     <%- if_p("ha_proxy.connections_rate_limit.block", "ha_proxy.connections_rate_limit.connections") do |block, connections| -%>
@@ -560,6 +562,8 @@ frontend https-in
   <%- if properties.ha_proxy.frontend_config -%>
     <%= format_indented_multiline_config(p("ha_proxy.frontend_config")) %>
   <%- end -%>
+    acl layer4_block src -f /var/vcap/jobs/haproxy/config/tcp_blacklist_cidrs.txt
+    tcp-request connection reject if layer4_block
   <%- if_p("ha_proxy.connections_rate_limit.table_size", "ha_proxy.connections_rate_limit.window_size") do -%>
     tcp-request connection track-sc0 src table st_tcp_conn_rate
     <%- if_p("ha_proxy.connections_rate_limit.block", "ha_proxy.connections_rate_limit.connections") do |block, connections| -%>

jobs/haproxy/templates/tcp_blacklist_cidrs.txt.erb

@@ -0,0 +1,20 @@
+# generated from tcp_blacklist_cidrs.txt.erb
+<%
+require "base64"
+require 'zlib'
+require 'stringio'
+
+cidrs = p("ha_proxy.tcp_blacklist_cidrs", [])
+uncompressed = ''
+if cidrs.is_a?(Array) && cidrs.any?
+  cidrs.each do |cidr|
+    uncompressed << cidr << "\n"
+  end
+elsif cidrs.is_a?(String)
+  gzplain = Base64.decode64(cidrs)
+  gz = Zlib::GzipReader.new(StringIO.new(gzplain))
+  uncompressed = gz.read
+end
+%>
+# This list contains CIDRs that are blocked immediately after TCP connection setup.
+<%= uncompressed %>

spec/haproxy/templates/haproxy_config/frontend_http_spec.rb

@@ -122,6 +122,39 @@
     end
   end
 
+  context 'TCP level blacklist (layer4_block)' do
+    context 'when ha_proxy.tcp_blacklist_cidrs is provided' do
+      let(:properties) do
+        { 'tcp_blacklist_cidrs' => ['172.168.4.1/32', '10.2.0.0/16'] }
+      end
+
+      it 'sets the correct acl and connection reject rules' do
+        expect(frontend_http).to include('acl layer4_block src -f /var/vcap/jobs/haproxy/config/tcp_blacklist_cidrs.txt')
+        expect(frontend_http).to include('tcp-request connection reject if layer4_block')
+      end
+    end
+
+    context 'when ha_proxy.tcp_blacklist_cidrs is not provided' do
+      let(:properties) { {} }
+
+      it 'still includes the layer4_block acl and reject rule (always present)' do
+        expect(frontend_http).to include('acl layer4_block src -f /var/vcap/jobs/haproxy/config/tcp_blacklist_cidrs.txt')
+        expect(frontend_http).to include('tcp-request connection reject if layer4_block')
+      end
+    end
+
+    context 'when ha_proxy.tcp_blacklist_cidrs is an empty array' do
+      let(:properties) do
+        { 'tcp_blacklist_cidrs' => [] }
+      end
+
+      it 'still includes the layer4_block acl and reject rule (always present)' do
+        expect(frontend_http).to include('acl layer4_block src -f /var/vcap/jobs/haproxy/config/tcp_blacklist_cidrs.txt')
+        expect(frontend_http).to include('tcp-request connection reject if layer4_block')
+      end
+    end
+  end
+
   context 'when ha_proxy.block_all is provided' do
     let(:properties) do
       { 'block_all' => true }

spec/haproxy/templates/haproxy_config/frontend_https_spec.rb

@@ -554,6 +554,39 @@
     end
   end
 
+  context 'TCP level blacklist (layer4_block)' do
+    context 'when ha_proxy.tcp_blacklist_cidrs is provided' do
+      let(:properties) do
+        default_properties.merge({ 'tcp_blacklist_cidrs' => ['172.168.4.1/32', '10.2.0.0/16'] })
+      end
+
+      it 'sets the correct acl and connection reject rules' do
+        expect(frontend_https).to include('acl layer4_block src -f /var/vcap/jobs/haproxy/config/tcp_blacklist_cidrs.txt')
+        expect(frontend_https).to include('tcp-request connection reject if layer4_block')
+      end
+    end
+
+    context 'when ha_proxy.tcp_blacklist_cidrs is not provided' do
+      let(:properties) { default_properties }
+
+      it 'still includes the layer4_block acl and reject rule (always present)' do
+        expect(frontend_https).to include('acl layer4_block src -f /var/vcap/jobs/haproxy/config/tcp_blacklist_cidrs.txt')
+        expect(frontend_https).to include('tcp-request connection reject if layer4_block')
+      end
+    end
+
+    context 'when ha_proxy.tcp_blacklist_cidrs is an empty array' do
+      let(:properties) do
+        default_properties.merge({ 'tcp_blacklist_cidrs' => [] })
+      end
+
+      it 'still includes the layer4_block acl and reject rule (always present)' do
+        expect(frontend_https).to include('acl layer4_block src -f /var/vcap/jobs/haproxy/config/tcp_blacklist_cidrs.txt')
+        expect(frontend_https).to include('tcp-request connection reject if layer4_block')
+      end
+    end
+  end
+
   context 'when ha_proxy.block_all is provided' do
     let(:properties) do
       default_properties.merge({ 'block_all' => true })

spec/haproxy/templates/tcp_blacklist_cidrs.txt_spec.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require 'rspec'
+
+describe 'config/tcp_blacklist_cidrs.txt' do
+  let(:template) { haproxy_job.template('config/tcp_blacklist_cidrs.txt') }
+
+  context 'when ha_proxy.tcp_blacklist_cidrs is provided' do
+    context 'when an array of cidrs is provided' do
+      it 'has the correct contents' do
+        expect(template.render({
+          'ha_proxy' => {
+            'tcp_blacklist_cidrs' => [
+              '10.0.0.0/8',
+              '192.168.2.0/24'
+            ]
+          }
+        })).to eq(<<~EXPECTED)
+          # generated from tcp_blacklist_cidrs.txt.erb
+
+          # This list contains CIDRs that are blocked immediately after TCP connection setup.
+          10.0.0.0/8
+          192.168.2.0/24
+
+        EXPECTED
+      end
+    end
+
+    context 'when a base64-encoded, gzipped config is provided' do
+      it 'has the correct contents' do
+        expect(template.render({
+          'ha_proxy' => {
+            'tcp_blacklist_cidrs' => gzip_and_b64_encode(<<~INPUT)
+              10.0.0.0/8
+              192.168.2.0/24
+            INPUT
+          }
+        })).to eq(<<~EXPECTED)
+          # generated from tcp_blacklist_cidrs.txt.erb
+
+          # This list contains CIDRs that are blocked immediately after TCP connection setup.
+          10.0.0.0/8
+          192.168.2.0/24
+
+        EXPECTED
+      end
+    end
+  end
+
+  context 'when ha_proxy.tcp_blacklist_cidrs is not provided' do
+    it 'contains only the default comment' do
+      expect(template.render({})).to eq(<<~EXPECTED)
+        # generated from tcp_blacklist_cidrs.txt.erb
+
+        # This list contains CIDRs that are blocked immediately after TCP connection setup.
+
+      EXPECTED
+    end
+  end
+
+  context 'when ha_proxy.tcp_blacklist_cidrs is an empty array' do
+    it 'contains only the default comment' do
+      expect(template.render({
+        'ha_proxy' => {
+          'tcp_blacklist_cidrs' => []
+        }
+      })).to eq(<<~EXPECTED)
+        # generated from tcp_blacklist_cidrs.txt.erb
+
+        # This list contains CIDRs that are blocked immediately after TCP connection setup.
+
+      EXPECTED
+    end
+  end
+end
+
+
+