From 1342801f6849673c91c257d9dc3d0d3e824d71d8 Mon Sep 17 00:00:00 2001 From: krzysztof baum Date: Mon, 2 Jan 2023 15:23:40 +0100 Subject: [PATCH 1/5] add gitattributes --- gcp/.gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 gcp/.gitattributes diff --git a/gcp/.gitattributes b/gcp/.gitattributes new file mode 100644 index 0000000..4eb2283 --- /dev/null +++ b/gcp/.gitattributes @@ -0,0 +1 @@ +text eol=lf \ No newline at end of file From 688fc5f2de891a27dce62ad0d867f2552b5b235a Mon Sep 17 00:00:00 2001 From: krzysztof baum Date: Mon, 2 Jan 2023 16:24:06 +0100 Subject: [PATCH 2/5] template based config --- gcp/.terraform.lock.hcl | 52 ++++++++ gcp/config/config.sh | 0 gcp/config/config.tpl | 109 +++++++++++++++++ gcp/config/squid.tpl | 43 +++++++ gcp/config/startup.sh | 155 +++++++++++++++++++----- gcp/config/whitelist.txt | 2 + gcp/ilb.tf | 10 +- gcp/images.tf | 7 +- gcp/main.tf | 43 +++++-- gcp/packer/solid-proxy-e2-small.pkr.hcl | 15 ++- gcp/terraform.tfvars | 5 + gcp/variables.tf | 11 +- 12 files changed, 394 insertions(+), 58 deletions(-) create mode 100644 gcp/config/config.sh create mode 100644 gcp/config/config.tpl create mode 100644 gcp/config/squid.tpl create mode 100644 gcp/config/whitelist.txt create mode 100644 gcp/terraform.tfvars diff --git a/gcp/.terraform.lock.hcl b/gcp/.terraform.lock.hcl index f0c4dfc..bc60610 100644 --- a/gcp/.terraform.lock.hcl +++ b/gcp/.terraform.lock.hcl @@ -20,3 +20,55 @@ provider "registry.terraform.io/hashicorp/google" { "zh:f9c75793eaeb2a34a12789359bff3d3c40d97c4f2f724d545aa997815dd5679f", ] } + +provider "registry.terraform.io/hashicorp/random" { + version = "3.4.3" + hashes = [ + "h1:xZGZf18JjMS06pFa4NErzANI98qi59SEcBsOcS2P2yQ=", + "zh:41c53ba47085d8261590990f8633c8906696fa0a3c4b384ff6a7ecbf84339752", + "zh:59d98081c4475f2ad77d881c4412c5129c56214892f490adf11c7e7a5a47de9b", + "zh:686ad1ee40b812b9e016317e7f34c0d63ef837e084dea4a1f578f64a6314ad53", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:84103eae7251384c0d995f5a257c72b0096605048f757b749b7b62107a5dccb3", + "zh:8ee974b110adb78c7cd18aae82b2729e5124d8f115d484215fd5199451053de5", + "zh:9dd4561e3c847e45de603f17fa0c01ae14cae8c4b7b4e6423c9ef3904b308dda", + "zh:bb07bb3c2c0296beba0beec629ebc6474c70732387477a65966483b5efabdbc6", + "zh:e891339e96c9e5a888727b45b2e1bb3fcbdfe0fd7c5b4396e4695459b38c8cb1", + "zh:ea4739860c24dfeaac6c100b2a2e357106a89d18751f7693f3c31ecf6a996f8d", + "zh:f0c76ac303fd0ab59146c39bc121c5d7d86f878e9a69294e29444d4c653786f8", + "zh:f143a9a5af42b38fed328a161279906759ff39ac428ebcfe55606e05e1518b93", + ] +} + +provider "registry.terraform.io/hashicorp/template" { + version = "2.2.0" + hashes = [ + "h1:94qn780bi1qjrbC3uQtjJh3Wkfwd5+tTtJHOb7KTg9w=", + "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", + "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", + "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", + "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16", + "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776", + "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451", + "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae", + "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde", + "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d", + "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2", + ] +} + +provider "registry.terraform.io/toowoxx/packer" { + version = "0.14.0" + hashes = [ + "h1:Y2ZjNZTRXH6TFgLkkJXR4zD44Rqo5FCQvmLnUZWBOSQ=", + "zh:0db1df78d1c576d22d2b1ffa71934103712d4dc885d7a27f39a3d0e99dd7b937", + "zh:39a3cb77bdd25a6f238cd59a35d1d486e29ec7c1ae0e6d8fd4b37b6ef5bf5e6c", + "zh:4f0de59babec8b5fe2d3f0aa1795043df68de56edf1a29ff2a29a55bc37e7487", + "zh:4faf5bec25eead65c172391f5bc8c6435f7251c2a784b1275cfc1f75f38b3b5a", + "zh:7a9913acc95e262a56e181fd97a95c848fdc92b036223a0f1b64e6bc9338d7ff", + "zh:90960b33ebb7df4ba16179d0b2739016f2c6f9d84b865e555975e2d3daa400e4", + "zh:9ef3018eec21fb9b6b46e6b0f925f6c588a96125d99c9aeac9f2646a5ed21df7", + "zh:ae539b4ee7b1d8ad598ba67a5685a44a05d747e97228d927dc4abc06276c48b6", + "zh:d09784ac984324c7c549d6671970b102b1bc25ad8d5efb426dcbf9ea36ffe6c1", + ] +} diff --git a/gcp/config/config.sh b/gcp/config/config.sh new file mode 100644 index 0000000..e69de29 diff --git a/gcp/config/config.tpl b/gcp/config/config.tpl new file mode 100644 index 0000000..a571e77 --- /dev/null +++ b/gcp/config/config.tpl @@ -0,0 +1,109 @@ +#!/bin/bash + +set -x # Debug +set -e # Strict exit codes + +echo "Applying system patches" +apt-get update +sudo apt-get upgrade -y +ufw disable + +echo "disabling iptables manual configuration" +echo iptables-persistent iptables-persistent/autosave_v4 boolean true | sudo debconf-set-selections +echo iptables-persistent iptables-persistent/autosave_v6 boolean true | sudo debconf-set-selections + +echo "Installing Ops Agent" +curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh +bash add-google-cloud-ops-agent-repo.sh --also-install + +echo "installing squid proxy server" +apt-get update +apt-get install squid3 libvirt-daemon-system apache2-utils iptables-persistent -y + +echo "configuring squid whitelists" + +echo "169.254.169.254 metadata.google.internal" >> /etc/hosts + +cat < /etc/squid/dst_whitelist.txt +${whitelist} +EOF + +cat < /etc/squid/src_whitelist.txt +metadata.google.internal +EOF + +echo "squid configuration" +cat < /etc/squid/squid.conf +${squid_config} +EOF + +echo "environment variables config" +SQUID_SERVER="192.168.1.1" +# Interface connected to Internet +INTERNET="eth0" +# Interface connected to LAN +LAN_IN="eth1" +# Squid port +SQUID_PORT="3128" +# Clean old firewall + +echo "configuring iptables" +iptables -F +iptables -X +iptables -t nat -F +iptables -t nat -X +iptables -t mangle -F +iptables -t mangle -X +echo 1 > /proc/sys/net/ipv4/ip_forward + +echo "Setting default filter policy" +iptables -P INPUT DROP +iptables -P OUTPUT ACCEPT +iptables -A INPUT -i lo -j ACCEPT +iptables -A OUTPUT -o lo -j ACCEPT +# Allow UDP, DNS and Passive FTP +iptables -A INPUT -i $INTERNET -m state --state ESTABLISHED,RELATED -j ACCEPT +# set this system as a router for Rest of LAN +iptables --table nat --append POSTROUTING --out-interface $INTERNET -j MASQUERADE +iptables --append FORWARD --in-interface $LAN_IN -j ACCEPT +# unlimited access to LAN +iptables -A INPUT -i $LAN_IN -j ACCEPT +iptables -A OUTPUT -o $LAN_IN -j ACCEPT +# DNAT port 80 request comming from LAN systems to squid 3128 ($SQUID_PORT) aka transparent proxy +iptables -t nat -A PREROUTING -i $LAN_IN -p tcp --dport 80 -j DNAT --to $SQUID_SERVER:$SQUID_PORT +iptables -t nat -A PREROUTING -i $INTERNET -p tcp --dport 80 -j REDIRECT --to-port $SQUID_PORT +# DROP everything and Log it +iptables -A INPUT -j LOG +iptables -A INPUT -j DROP + +echo "restarting squid" +service squid restart + +echo "saving iptables" +/sbin/iptables-save + +echo "Configure Ops Agent log streams" +# TODO + +echo "Healthcheck API" +# TODO + + +SQUID_SERVER="127.0.0.1" +# Interface connected to Internet +INTERNET="ens4" +# Interface connected to LAN +LAN_IN="lo" +# Squid port +SQUID_PORT="3128" +# Clean old firewall +systemctl start libvirtd + + +echo "environment variables config" +echo "Configure Ops Agent log streams" +# TODO + +echo "Healthcheck API" +# TODO + diff --git a/gcp/config/squid.tpl b/gcp/config/squid.tpl new file mode 100644 index 0000000..a663188 --- /dev/null +++ b/gcp/config/squid.tpl @@ -0,0 +1,43 @@ +hierarchy_stoplist cgi-bin ? +acl QUERY urlpath_regex cgi-bin \? +no_cache deny QUERY +hosts_file /etc/hosts +refresh_pattern ^ftp: 1440 20% 10080 +refresh_pattern ^gopher: 1440 0% 1440 +refresh_pattern . 0 20% 4320 +acl all src 0.0.0.0/0.0.0.0 +acl manager proto cache_object +acl localhost src 127.0.0.1/255.255.255.255 +acl to_localhost dst 127.0.0.0/8 +acl purge method PURGE +acl CONNECT method CONNECT + +# connection to http and ssl +acl Safe_ports port 80 +acl Safe_ports port 443 +acl SSL_ports port 22 + +cache_mem 1024 MB +acl src_whitelist srcdomain "/etc/squid/src_whitelist.txt" +acl dst_whitelist dstdomain "/etc/squid/dst_whitelist.txt" + +http_access allow src_whitelist +http_access allow dst_whitelist +http_access allow CONNECT +http_access deny CONNECT !SSL_ports +http_access allow manager localhost +http_access deny manager +http_access allow purge localhost +http_access deny purge +http_access deny !Safe_ports +acl lan src ${cidr} +acl my_phisical src 31.182.219.133 + +http_access allow localhost +http_access allow lan +http_access allow my_phisical +http_access deny all +http_reply_access allow all +icp_access allow all +http_port 3128 transparent +coredump_dir /var/spool/squid \ No newline at end of file diff --git a/gcp/config/startup.sh b/gcp/config/startup.sh index d89e0ff..c98640c 100644 --- a/gcp/config/startup.sh +++ b/gcp/config/startup.sh @@ -4,49 +4,142 @@ set -x # Debug set -e # Strict exit codes echo "Applying system patches" -sudo apt-get update; +apt-get update +sudo apt-get upgrade -y +ufw disable -sudo apt-get upgrade -y; +echo "disabling iptables manual configuration" +echo iptables-persistent iptables-persistent/autosave_v4 boolean true | sudo debconf-set-selections +echo iptables-persistent iptables-persistent/autosave_v6 boolean true | sudo debconf-set-selections +echo "Installing Ops Agent" +curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh +bash add-google-cloud-ops-agent-repo.sh --also-install -echo "Installing Ops Agent"; -curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh; -bash add-google-cloud-ops-agent-repo.sh --also-install ; +echo "installing squid proxy server" +apt-get update +apt-get install squid3 libvirt-daemon-system apache2-utils iptables-persistent -y +echo "configuring squid whitelists" +cat < /etc/squid/domain_whitelist.txt +metadata.google.internal +www.google.com +EOF -echo "Installing wget"; -sudo apt-get install wget -y; +cat < /etc/squid/src_whitelist.txt +metadata.google.internal +EOF -echo "Installing MITM Proxy"; -if [[ $instance_name =~ ^packer.*] ]; -then -sudo wget https://snapshots.mitmproxy.org/8.1.1/mitmproxy-8.1.1-linux.tar.gz ; -tar -zxvf mitmproxy-8.1.1-linux.tar.gz; -rm mitmproxy-8.1.1-linux.tar.gz; -./mitmproxy --mode transparent --showhost; -fi +echo "minimal configuration" +cat < /etc/squid/squid.conf +hierarchy_stoplist cgi-bin ? +acl QUERY urlpath_regex cgi-bin \? +no_cache deny QUERY +hosts_file /etc/hosts +refresh_pattern ^ftp: 1440 20% 10080 +refresh_pattern ^gopher: 1440 0% 1440 +refresh_pattern . 0 20% 4320 +acl all src 0.0.0.0/0.0.0.0 +acl manager proto cache_object +acl localhost src 127.0.0.1/255.255.255.255 +acl to_localhost dst 127.0.0.0/8 +acl purge method PURGE +acl CONNECT method CONNECT -echo "Apply MITM configuration"; -echo "enable forwarding mode"; -sudo sysctl -w net.ipv4.ip_forward=1; -sudo sysctl -w net.ipv6.conf.all.forwarding=1; +# connection to http and ssl +acl Safe_ports port 80 +acl Safe_ports port 443 +acl SSL_ports port 22 -echo "disable ICMP redirects"; -sudo sysctl -w net.ipv4.conf.all.send_redirects=0; +cache_mem 1024 MB +acl src_whitelist srcdomain "/etc/squid/src_whitelist.txt" +acl dst_whitelist dstdomain "/etc/squid/dst_whitelist.txt" -echo "configure iptables"; +http_access allow src_whitelist +http_access all src_whitelist +http_access allow CONNECT +http_access deny CONNECT !SSL_ports +http_access allow manager localhost +http_access deny manager +http_access allow purge localhost +http_access deny purge +http_access deny !Safe_ports +acl lan src 10.132.0.0/20 +acl my_phisical src 31.182.219.133 -sudo iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080; -sudo iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8080; -sudo ip6tables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080; -sudo ip6tables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8080; +http_access allow localhost +http_access allow lan +http_access allow my_phisical +http_access deny all +http_reply_access allow all +icp_access allow all +http_port 3128 transparent +coredump_dir /var/spool/squid +EOF -echo "start mitmproxy if not in packer"; -instance_name=$(curl http://metadata.google.internal/computeMetadata/v1/instance/id -H "Metadata-Flavor: Google") -if [[ ! $instance_name =~ ^packer.*] ]; then ./mitmproxy --mode transparent --showhost; fi +echo "environment variables config" +SQUID_SERVER="192.168.1.1" +# Interface connected to Internet +INTERNET="eth0" +# Interface connected to LAN +LAN_IN="eth1" +# Squid port +SQUID_PORT="3128" +# Clean old firewall -echo "Configure Ops Agent log streams"; +echo "configuring iptables" + +iptables -F +iptables -X +iptables -t nat -F +iptables -t nat -X +iptables -t mangle -F +iptables -t mangle -X +echo 1 > /proc/sys/net/ipv4/ip_forward +echo "Setting default filter policy" +iptables -P INPUT DROP +iptables -P OUTPUT ACCEPT +iptables -A INPUT -i lo -j ACCEPT +iptables -A OUTPUT -o lo -j ACCEPT +# Allow UDP, DNS and Passive FTP +iptables -A INPUT -i $INTERNET -m state --state ESTABLISHED,RELATED -j ACCEPT +# set this system as a router for Rest of LAN +iptables --table nat --append POSTROUTING --out-interface $INTERNET -j MASQUERADE +iptables --append FORWARD --in-interface $LAN_IN -j ACCEPT +# unlimited access to LAN +iptables -A INPUT -i $LAN_IN -j ACCEPT +iptables -A OUTPUT -o $LAN_IN -j ACCEPT +# DNAT port 80 request comming from LAN systems to squid 3128 ($SQUID_PORT) aka transparent proxy +iptables -t nat -A PREROUTING -i $LAN_IN -p tcp --dport 80 -j DNAT --to $SQUID_SERVER:$SQUID_PORT +iptables -t nat -A PREROUTING -i $INTERNET -p tcp --dport 80 -j REDIRECT --to-port $SQUID_PORT +# DROP everything and Log it +iptables -A INPUT -j LOG +iptables -A INPUT -j DROP +echo "restarting squid" +service squid restart +echo "saving iptables" +/sbin/iptables-save +echo "Configure Ops Agent log streams" # TODO -echo "Healthcheck API"; +echo "Healthcheck API" # TODO + + +SQUID_SERVER="127.0.0.1" +# Interface connected to Internet +INTERNET="ens4" +# Interface connected to LAN +LAN_IN="lo" +# Squid port +SQUID_PORT="3128" +# Clean old firewall +systemctl start libvirtd + + +echo "environment variables config" +echo "Configure Ops Agent log streams" +# TODO + +echo "Healthcheck API" +# TODO \ No newline at end of file diff --git a/gcp/config/whitelist.txt b/gcp/config/whitelist.txt new file mode 100644 index 0000000..0993fa0 --- /dev/null +++ b/gcp/config/whitelist.txt @@ -0,0 +1,2 @@ +metadata.google.internal +www.google.com \ No newline at end of file diff --git a/gcp/ilb.tf b/gcp/ilb.tf index 6e5ff42..b2027df 100644 --- a/gcp/ilb.tf +++ b/gcp/ilb.tf @@ -2,16 +2,14 @@ resource "google_compute_forwarding_rule" "egress_filter_forwarding_rule" { name = "egress-filter-forwarding-rule" backend_service = google_compute_region_backend_service.egress_filter_backend.id - region = var.region - ip_protocol = "TCP" load_balancing_scheme = "INTERNAL" all_ports = true allow_global_access = true - - network = var.vpc_name - subnetwork = var.subnet_name + project = var.project + network = var.vpc_name + subnetwork = var.subnet_name } # backend service @@ -39,6 +37,6 @@ resource "google_compute_region_health_check" "egress_filter" { unhealthy_threshold = 10 tcp_health_check { - port = 8080 + port = 80 } } \ No newline at end of file diff --git a/gcp/images.tf b/gcp/images.tf index 3a4e528..bfd9193 100644 --- a/gcp/images.tf +++ b/gcp/images.tf @@ -5,8 +5,8 @@ data "packer_version" "version" {} data "packer_files" "packer_config_dir" { directory = "${path.module}/packer" } -data "packer_files" "vm_config_files"{ - directory = "${path.module}/config" +data "packer_files" "vm_config_files" { + directory = "${path.module}/config" } @@ -36,5 +36,6 @@ output "packer_version" { } output "image_id" { - value = packer_image.solid_proxy_image.id + value = packer_image.solid_proxy_image.id } + diff --git a/gcp/main.tf b/gcp/main.tf index 1ae263b..7814562 100644 --- a/gcp/main.tf +++ b/gcp/main.tf @@ -1,3 +1,27 @@ + +data "google_compute_subnetwork" "subnet" { + name = var.subnet_name + region = var.region + project = var.project +} +data "template_file" "squid_config" { + template = file("${path.root}/config/squid.tpl") + vars = { + cidr = data.google_compute_subnetwork.subnet.ip_cidr_range + } +} + +data "template_file" "startup_script" { + template = file("${path.root}/config/config.tpl") + vars = { + squid_config = data.template_file.squid_config.rendered + whitelist = file("${path.root}/config/whitelist.txt") + } +} +output "startup_script" { + value = data.template_file.startup_script.rendered +} + resource "google_compute_instance_template" "egress_filter" { project = var.project name_prefix = "${local.prefix}egress-filter" @@ -14,7 +38,7 @@ resource "google_compute_instance_template" "egress_filter" { auto_delete = false - source_image = "packer-1663596114" + source_image = "ubuntu-1804-bionic-arm64-v20221201" # TODO KMS support # disk_encryption_key { # } @@ -22,7 +46,7 @@ resource "google_compute_instance_template" "egress_filter" { network_interface { network = var.vpc_name - subnetwork = var.subnet_name + subnetwork = data.google_compute_subnetwork.subnet.self_link } can_ip_forward = true @@ -32,9 +56,7 @@ resource "google_compute_instance_template" "egress_filter" { metadata = { # TODO - base image from packer or with startup script - startup-script = templatefile("${path.module}/config/startup.sh", { - - }) + startup-script = data.template_file.startup_script.rendered } # TODO @@ -60,13 +82,17 @@ resource "google_compute_region_instance_group_manager" "egress_filter" { name = "egress-filter-igm" base_instance_name = "egress-filter" - region = var.region + region = var.region - target_pools = [ google_compute_target_pool.egress_filter.id ] + target_pools = [google_compute_target_pool.egress_filter.id] version { instance_template = google_compute_instance_template.egress_filter.id } + named_port { + name = "http" + port = 80 + } } resource "google_compute_target_pool" "egress_filter" { @@ -81,7 +107,7 @@ resource "google_compute_region_autoscaler" "egress_filter" { target = google_compute_region_instance_group_manager.egress_filter.id autoscaling_policy { - mode = var.autoscaling_mode + mode = var.autoscaling_mode max_replicas = var.autoscaling_max_replicas min_replicas = var.autoscaling_min_replicas cooldown_period = var.autoscaling_cooldown_period @@ -91,3 +117,4 @@ resource "google_compute_region_autoscaler" "egress_filter" { } } } + diff --git a/gcp/packer/solid-proxy-e2-small.pkr.hcl b/gcp/packer/solid-proxy-e2-small.pkr.hcl index f6385b5..a945851 100644 --- a/gcp/packer/solid-proxy-e2-small.pkr.hcl +++ b/gcp/packer/solid-proxy-e2-small.pkr.hcl @@ -1,18 +1,21 @@ -source "googlecompute" "mitm-proxy" { +source "googlecompute" "squid-proxy" { machine_type = "e2-standard-4" project_id = "solid-egress-filter" source_image_family = "ubuntu-1804-lts" ssh_username = "solid-egress-filter" startup_script_file = "${path.root}/../config/startup.sh" + network = "projects/solid-egress-filter/global/networks/dev-vpc" + subnetwork = "projects/solid-egress-filter/regions/europe-west1/subnetworks/dev-subnet" zone = "europe-west1-b" } build { name = "solid-proxy-disk" - sources = ["source.googlecompute.mitm-proxy"] - # provisioner "file"{ - # source = "" - # destination ="/home/solid-egres-filter/mitm-config" - # } + sources = ["source.googlecompute.squid-proxy"] + + provisioner "file"{ + source = "" + destination ="/home/solid-egres-filter/squid-config" + } } diff --git a/gcp/terraform.tfvars b/gcp/terraform.tfvars new file mode 100644 index 0000000..ac5c0ab --- /dev/null +++ b/gcp/terraform.tfvars @@ -0,0 +1,5 @@ +project = "solid-egress-filter" +region = "europe-west1" +prefix = "" + +subnet_link = "projects/solid-egress-filter/regions/europe-west1/subnetworks/default" \ No newline at end of file diff --git a/gcp/variables.tf b/gcp/variables.tf index a87534c..6e56c31 100644 --- a/gcp/variables.tf +++ b/gcp/variables.tf @@ -17,6 +17,9 @@ variable "subnet_name" { type = string default = "default" } +variable "subnet_link" { + type = string +} variable "prefix" { description = "Prefix for all resource names. Allows to deploy multiple instances in the same project." @@ -49,21 +52,21 @@ variable "sa_name" { } variable "autoscaling_mode" { - type = string + type = string default = "ON" } variable "autoscaling_min_replicas" { - type = number + type = number default = 1 } variable "autoscaling_max_replicas" { - type = number + type = number default = 3 } variable "autoscaling_cooldown_period" { - type = number + type = number default = 300 } From a36ed0619e40efe8540cc9e3f3a357ab88e4103c Mon Sep 17 00:00:00 2001 From: krzysztof baum Date: Mon, 2 Jan 2023 16:36:11 +0100 Subject: [PATCH 3/5] remove unused variable --- gcp/terraform.tfvars | 2 -- gcp/variables.tf | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/gcp/terraform.tfvars b/gcp/terraform.tfvars index ac5c0ab..01bbe9d 100644 --- a/gcp/terraform.tfvars +++ b/gcp/terraform.tfvars @@ -1,5 +1,3 @@ project = "solid-egress-filter" region = "europe-west1" prefix = "" - -subnet_link = "projects/solid-egress-filter/regions/europe-west1/subnetworks/default" \ No newline at end of file diff --git a/gcp/variables.tf b/gcp/variables.tf index 6e56c31..e9e5288 100644 --- a/gcp/variables.tf +++ b/gcp/variables.tf @@ -17,9 +17,7 @@ variable "subnet_name" { type = string default = "default" } -variable "subnet_link" { - type = string -} + variable "prefix" { description = "Prefix for all resource names. Allows to deploy multiple instances in the same project." From fdd3a276c5f805fc994c888a634b5cfe23254ea7 Mon Sep 17 00:00:00 2001 From: krzysztof baum Date: Mon, 2 Jan 2023 16:39:03 +0100 Subject: [PATCH 4/5] replace path to root with path to module --- gcp/main.tf | 6 +++--- gcp/packer/solid-proxy-e2-small.pkr.hcl | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gcp/main.tf b/gcp/main.tf index 7814562..dfbcfb8 100644 --- a/gcp/main.tf +++ b/gcp/main.tf @@ -5,17 +5,17 @@ data "google_compute_subnetwork" "subnet" { project = var.project } data "template_file" "squid_config" { - template = file("${path.root}/config/squid.tpl") + template = file("${path.module}/config/squid.tpl") vars = { cidr = data.google_compute_subnetwork.subnet.ip_cidr_range } } data "template_file" "startup_script" { - template = file("${path.root}/config/config.tpl") + template = file("${path.module}/config/config.tpl") vars = { squid_config = data.template_file.squid_config.rendered - whitelist = file("${path.root}/config/whitelist.txt") + whitelist = file("${path.module}/config/whitelist.txt") } } output "startup_script" { diff --git a/gcp/packer/solid-proxy-e2-small.pkr.hcl b/gcp/packer/solid-proxy-e2-small.pkr.hcl index a945851..ca93ba4 100644 --- a/gcp/packer/solid-proxy-e2-small.pkr.hcl +++ b/gcp/packer/solid-proxy-e2-small.pkr.hcl @@ -4,7 +4,7 @@ source "googlecompute" "squid-proxy" { project_id = "solid-egress-filter" source_image_family = "ubuntu-1804-lts" ssh_username = "solid-egress-filter" - startup_script_file = "${path.root}/../config/startup.sh" + startup_script_file = "${path.module}/../config/startup.sh" network = "projects/solid-egress-filter/global/networks/dev-vpc" subnetwork = "projects/solid-egress-filter/regions/europe-west1/subnetworks/dev-subnet" zone = "europe-west1-b" From 146201005db1d49b7e228d951dc85e80710eb94b Mon Sep 17 00:00:00 2001 From: krzysztof baum Date: Mon, 2 Jan 2023 19:31:48 +0100 Subject: [PATCH 5/5] remove unnesesary lines --- gcp/config/config.tpl | 56 ++++++++-------- gcp/config/squid.tpl | 20 +----- gcp/config/startup.sh | 145 ------------------------------------------ 3 files changed, 31 insertions(+), 190 deletions(-) delete mode 100644 gcp/config/startup.sh diff --git a/gcp/config/config.tpl b/gcp/config/config.tpl index a571e77..c65341b 100644 --- a/gcp/config/config.tpl +++ b/gcp/config/config.tpl @@ -47,34 +47,34 @@ LAN_IN="eth1" SQUID_PORT="3128" # Clean old firewall -echo "configuring iptables" -iptables -F -iptables -X -iptables -t nat -F -iptables -t nat -X -iptables -t mangle -F -iptables -t mangle -X -echo 1 > /proc/sys/net/ipv4/ip_forward - -echo "Setting default filter policy" -iptables -P INPUT DROP -iptables -P OUTPUT ACCEPT -iptables -A INPUT -i lo -j ACCEPT -iptables -A OUTPUT -o lo -j ACCEPT -# Allow UDP, DNS and Passive FTP -iptables -A INPUT -i $INTERNET -m state --state ESTABLISHED,RELATED -j ACCEPT -# set this system as a router for Rest of LAN -iptables --table nat --append POSTROUTING --out-interface $INTERNET -j MASQUERADE -iptables --append FORWARD --in-interface $LAN_IN -j ACCEPT -# unlimited access to LAN -iptables -A INPUT -i $LAN_IN -j ACCEPT -iptables -A OUTPUT -o $LAN_IN -j ACCEPT -# DNAT port 80 request comming from LAN systems to squid 3128 ($SQUID_PORT) aka transparent proxy -iptables -t nat -A PREROUTING -i $LAN_IN -p tcp --dport 80 -j DNAT --to $SQUID_SERVER:$SQUID_PORT -iptables -t nat -A PREROUTING -i $INTERNET -p tcp --dport 80 -j REDIRECT --to-port $SQUID_PORT -# DROP everything and Log it -iptables -A INPUT -j LOG -iptables -A INPUT -j DROP +# echo "configuring iptables" +# iptables -F +# iptables -X +# iptables -t nat -F +# iptables -t nat -X +# iptables -t mangle -F +# iptables -t mangle -X +# echo 1 > /proc/sys/net/ipv4/ip_forward + +# echo "Setting default filter policy" +# iptables -P INPUT DROP +# iptables -P OUTPUT ACCEPT +# iptables -A INPUT -i lo -j ACCEPT +# iptables -A OUTPUT -o lo -j ACCEPT +# # Allow UDP, DNS and Passive FTP +# iptables -A INPUT -i $INTERNET -m state --state ESTABLISHED,RELATED -j ACCEPT +# # set this system as a router for Rest of LAN +# iptables --table nat --append POSTROUTING --out-interface $INTERNET -j MASQUERADE +# iptables --append FORWARD --in-interface $LAN_IN -j ACCEPT +# # unlimited access to LAN +# iptables -A INPUT -i $LAN_IN -j ACCEPT +# iptables -A OUTPUT -o $LAN_IN -j ACCEPT +# # DNAT port 80 request comming from LAN systems to squid 3128 ($SQUID_PORT) aka transparent proxy +# iptables -t nat -A PREROUTING -i $LAN_IN -p tcp --dport 80 -j DNAT --to $SQUID_SERVER:$SQUID_PORT +# iptables -t nat -A PREROUTING -i $INTERNET -p tcp --dport 80 -j REDIRECT --to-port $SQUID_PORT +# # DROP everything and Log it +# iptables -A INPUT -j LOG +# iptables -A INPUT -j DROP echo "restarting squid" service squid restart diff --git a/gcp/config/squid.tpl b/gcp/config/squid.tpl index a663188..07bf3dc 100644 --- a/gcp/config/squid.tpl +++ b/gcp/config/squid.tpl @@ -1,43 +1,29 @@ -hierarchy_stoplist cgi-bin ? -acl QUERY urlpath_regex cgi-bin \? -no_cache deny QUERY hosts_file /etc/hosts -refresh_pattern ^ftp: 1440 20% 10080 -refresh_pattern ^gopher: 1440 0% 1440 -refresh_pattern . 0 20% 4320 acl all src 0.0.0.0/0.0.0.0 -acl manager proto cache_object acl localhost src 127.0.0.1/255.255.255.255 acl to_localhost dst 127.0.0.0/8 -acl purge method PURGE acl CONNECT method CONNECT -# connection to http and ssl +# connection to http, https and ssl acl Safe_ports port 80 acl Safe_ports port 443 acl SSL_ports port 22 -cache_mem 1024 MB acl src_whitelist srcdomain "/etc/squid/src_whitelist.txt" acl dst_whitelist dstdomain "/etc/squid/dst_whitelist.txt" +acl lan src ${cidr} +acl my_phisical src 31.182.219.133 http_access allow src_whitelist http_access allow dst_whitelist http_access allow CONNECT http_access deny CONNECT !SSL_ports -http_access allow manager localhost -http_access deny manager -http_access allow purge localhost -http_access deny purge http_access deny !Safe_ports -acl lan src ${cidr} -acl my_phisical src 31.182.219.133 http_access allow localhost http_access allow lan http_access allow my_phisical http_access deny all http_reply_access allow all -icp_access allow all http_port 3128 transparent coredump_dir /var/spool/squid \ No newline at end of file diff --git a/gcp/config/startup.sh b/gcp/config/startup.sh deleted file mode 100644 index c98640c..0000000 --- a/gcp/config/startup.sh +++ /dev/null @@ -1,145 +0,0 @@ -#!/bin/bash - -set -x # Debug -set -e # Strict exit codes - -echo "Applying system patches" -apt-get update -sudo apt-get upgrade -y -ufw disable - -echo "disabling iptables manual configuration" -echo iptables-persistent iptables-persistent/autosave_v4 boolean true | sudo debconf-set-selections -echo iptables-persistent iptables-persistent/autosave_v6 boolean true | sudo debconf-set-selections - -echo "Installing Ops Agent" -curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh -bash add-google-cloud-ops-agent-repo.sh --also-install - -echo "installing squid proxy server" -apt-get update -apt-get install squid3 libvirt-daemon-system apache2-utils iptables-persistent -y -echo "configuring squid whitelists" -cat < /etc/squid/domain_whitelist.txt -metadata.google.internal -www.google.com -EOF - -cat < /etc/squid/src_whitelist.txt -metadata.google.internal -EOF - -echo "minimal configuration" -cat < /etc/squid/squid.conf -hierarchy_stoplist cgi-bin ? -acl QUERY urlpath_regex cgi-bin \? -no_cache deny QUERY -hosts_file /etc/hosts -refresh_pattern ^ftp: 1440 20% 10080 -refresh_pattern ^gopher: 1440 0% 1440 -refresh_pattern . 0 20% 4320 -acl all src 0.0.0.0/0.0.0.0 -acl manager proto cache_object -acl localhost src 127.0.0.1/255.255.255.255 -acl to_localhost dst 127.0.0.0/8 -acl purge method PURGE -acl CONNECT method CONNECT - -# connection to http and ssl -acl Safe_ports port 80 -acl Safe_ports port 443 -acl SSL_ports port 22 - -cache_mem 1024 MB -acl src_whitelist srcdomain "/etc/squid/src_whitelist.txt" -acl dst_whitelist dstdomain "/etc/squid/dst_whitelist.txt" - -http_access allow src_whitelist -http_access all src_whitelist -http_access allow CONNECT -http_access deny CONNECT !SSL_ports -http_access allow manager localhost -http_access deny manager -http_access allow purge localhost -http_access deny purge -http_access deny !Safe_ports -acl lan src 10.132.0.0/20 -acl my_phisical src 31.182.219.133 - -http_access allow localhost -http_access allow lan -http_access allow my_phisical -http_access deny all -http_reply_access allow all -icp_access allow all -http_port 3128 transparent -coredump_dir /var/spool/squid -EOF - -echo "environment variables config" - -SQUID_SERVER="192.168.1.1" -# Interface connected to Internet -INTERNET="eth0" -# Interface connected to LAN -LAN_IN="eth1" -# Squid port -SQUID_PORT="3128" -# Clean old firewall - -echo "configuring iptables" - -iptables -F -iptables -X -iptables -t nat -F -iptables -t nat -X -iptables -t mangle -F -iptables -t mangle -X -echo 1 > /proc/sys/net/ipv4/ip_forward -echo "Setting default filter policy" -iptables -P INPUT DROP -iptables -P OUTPUT ACCEPT -iptables -A INPUT -i lo -j ACCEPT -iptables -A OUTPUT -o lo -j ACCEPT -# Allow UDP, DNS and Passive FTP -iptables -A INPUT -i $INTERNET -m state --state ESTABLISHED,RELATED -j ACCEPT -# set this system as a router for Rest of LAN -iptables --table nat --append POSTROUTING --out-interface $INTERNET -j MASQUERADE -iptables --append FORWARD --in-interface $LAN_IN -j ACCEPT -# unlimited access to LAN -iptables -A INPUT -i $LAN_IN -j ACCEPT -iptables -A OUTPUT -o $LAN_IN -j ACCEPT -# DNAT port 80 request comming from LAN systems to squid 3128 ($SQUID_PORT) aka transparent proxy -iptables -t nat -A PREROUTING -i $LAN_IN -p tcp --dport 80 -j DNAT --to $SQUID_SERVER:$SQUID_PORT -iptables -t nat -A PREROUTING -i $INTERNET -p tcp --dport 80 -j REDIRECT --to-port $SQUID_PORT -# DROP everything and Log it -iptables -A INPUT -j LOG -iptables -A INPUT -j DROP -echo "restarting squid" -service squid restart -echo "saving iptables" -/sbin/iptables-save -echo "Configure Ops Agent log streams" -# TODO - -echo "Healthcheck API" -# TODO - - -SQUID_SERVER="127.0.0.1" -# Interface connected to Internet -INTERNET="ens4" -# Interface connected to LAN -LAN_IN="lo" -# Squid port -SQUID_PORT="3128" -# Clean old firewall -systemctl start libvirtd - - -echo "environment variables config" -echo "Configure Ops Agent log streams" -# TODO - -echo "Healthcheck API" -# TODO \ No newline at end of file