diff --git a/gcp/.gitattributes b/gcp/.gitattributes new file mode 100644 index 0000000..4eb2283 --- /dev/null +++ b/gcp/.gitattributes @@ -0,0 +1 @@ +text eol=lf \ No newline at end of file diff --git a/gcp/.terraform.lock.hcl b/gcp/.terraform.lock.hcl index f0c4dfc..bc60610 100644 --- a/gcp/.terraform.lock.hcl +++ b/gcp/.terraform.lock.hcl @@ -20,3 +20,55 @@ provider "registry.terraform.io/hashicorp/google" { "zh:f9c75793eaeb2a34a12789359bff3d3c40d97c4f2f724d545aa997815dd5679f", ] } + +provider "registry.terraform.io/hashicorp/random" { + version = "3.4.3" + hashes = [ + "h1:xZGZf18JjMS06pFa4NErzANI98qi59SEcBsOcS2P2yQ=", + "zh:41c53ba47085d8261590990f8633c8906696fa0a3c4b384ff6a7ecbf84339752", + "zh:59d98081c4475f2ad77d881c4412c5129c56214892f490adf11c7e7a5a47de9b", + "zh:686ad1ee40b812b9e016317e7f34c0d63ef837e084dea4a1f578f64a6314ad53", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:84103eae7251384c0d995f5a257c72b0096605048f757b749b7b62107a5dccb3", + "zh:8ee974b110adb78c7cd18aae82b2729e5124d8f115d484215fd5199451053de5", + "zh:9dd4561e3c847e45de603f17fa0c01ae14cae8c4b7b4e6423c9ef3904b308dda", + "zh:bb07bb3c2c0296beba0beec629ebc6474c70732387477a65966483b5efabdbc6", + "zh:e891339e96c9e5a888727b45b2e1bb3fcbdfe0fd7c5b4396e4695459b38c8cb1", + "zh:ea4739860c24dfeaac6c100b2a2e357106a89d18751f7693f3c31ecf6a996f8d", + "zh:f0c76ac303fd0ab59146c39bc121c5d7d86f878e9a69294e29444d4c653786f8", + "zh:f143a9a5af42b38fed328a161279906759ff39ac428ebcfe55606e05e1518b93", + ] +} + +provider "registry.terraform.io/hashicorp/template" { + version = "2.2.0" + hashes = [ + "h1:94qn780bi1qjrbC3uQtjJh3Wkfwd5+tTtJHOb7KTg9w=", + "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", + "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", + "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", + "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16", + "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776", + "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451", + "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae", + "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde", + "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d", + "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2", + ] +} + +provider "registry.terraform.io/toowoxx/packer" { + version = "0.14.0" + hashes = [ + "h1:Y2ZjNZTRXH6TFgLkkJXR4zD44Rqo5FCQvmLnUZWBOSQ=", + "zh:0db1df78d1c576d22d2b1ffa71934103712d4dc885d7a27f39a3d0e99dd7b937", + "zh:39a3cb77bdd25a6f238cd59a35d1d486e29ec7c1ae0e6d8fd4b37b6ef5bf5e6c", + "zh:4f0de59babec8b5fe2d3f0aa1795043df68de56edf1a29ff2a29a55bc37e7487", + "zh:4faf5bec25eead65c172391f5bc8c6435f7251c2a784b1275cfc1f75f38b3b5a", + "zh:7a9913acc95e262a56e181fd97a95c848fdc92b036223a0f1b64e6bc9338d7ff", + "zh:90960b33ebb7df4ba16179d0b2739016f2c6f9d84b865e555975e2d3daa400e4", + "zh:9ef3018eec21fb9b6b46e6b0f925f6c588a96125d99c9aeac9f2646a5ed21df7", + "zh:ae539b4ee7b1d8ad598ba67a5685a44a05d747e97228d927dc4abc06276c48b6", + "zh:d09784ac984324c7c549d6671970b102b1bc25ad8d5efb426dcbf9ea36ffe6c1", + ] +} diff --git a/gcp/config/config.sh b/gcp/config/config.sh new file mode 100644 index 0000000..e69de29 diff --git a/gcp/config/config.tpl b/gcp/config/config.tpl new file mode 100644 index 0000000..c65341b --- /dev/null +++ b/gcp/config/config.tpl @@ -0,0 +1,109 @@ +#!/bin/bash + +set -x # Debug +set -e # Strict exit codes + +echo "Applying system patches" +apt-get update +sudo apt-get upgrade -y +ufw disable + +echo "disabling iptables manual configuration" +echo iptables-persistent iptables-persistent/autosave_v4 boolean true | sudo debconf-set-selections +echo iptables-persistent iptables-persistent/autosave_v6 boolean true | sudo debconf-set-selections + +echo "Installing Ops Agent" +curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh +bash add-google-cloud-ops-agent-repo.sh --also-install + +echo "installing squid proxy server" +apt-get update +apt-get install squid3 libvirt-daemon-system apache2-utils iptables-persistent -y + +echo "configuring squid whitelists" + +echo "169.254.169.254 metadata.google.internal" >> /etc/hosts + +cat < /etc/squid/dst_whitelist.txt +${whitelist} +EOF + +cat < /etc/squid/src_whitelist.txt +metadata.google.internal +EOF + +echo "squid configuration" +cat < /etc/squid/squid.conf +${squid_config} +EOF + +echo "environment variables config" +SQUID_SERVER="192.168.1.1" +# Interface connected to Internet +INTERNET="eth0" +# Interface connected to LAN +LAN_IN="eth1" +# Squid port +SQUID_PORT="3128" +# Clean old firewall + +# echo "configuring iptables" +# iptables -F +# iptables -X +# iptables -t nat -F +# iptables -t nat -X +# iptables -t mangle -F +# iptables -t mangle -X +# echo 1 > /proc/sys/net/ipv4/ip_forward + +# echo "Setting default filter policy" +# iptables -P INPUT DROP +# iptables -P OUTPUT ACCEPT +# iptables -A INPUT -i lo -j ACCEPT +# iptables -A OUTPUT -o lo -j ACCEPT +# # Allow UDP, DNS and Passive FTP +# iptables -A INPUT -i $INTERNET -m state --state ESTABLISHED,RELATED -j ACCEPT +# # set this system as a router for Rest of LAN +# iptables --table nat --append POSTROUTING --out-interface $INTERNET -j MASQUERADE +# iptables --append FORWARD --in-interface $LAN_IN -j ACCEPT +# # unlimited access to LAN +# iptables -A INPUT -i $LAN_IN -j ACCEPT +# iptables -A OUTPUT -o $LAN_IN -j ACCEPT +# # DNAT port 80 request comming from LAN systems to squid 3128 ($SQUID_PORT) aka transparent proxy +# iptables -t nat -A PREROUTING -i $LAN_IN -p tcp --dport 80 -j DNAT --to $SQUID_SERVER:$SQUID_PORT +# iptables -t nat -A PREROUTING -i $INTERNET -p tcp --dport 80 -j REDIRECT --to-port $SQUID_PORT +# # DROP everything and Log it +# iptables -A INPUT -j LOG +# iptables -A INPUT -j DROP + +echo "restarting squid" +service squid restart + +echo "saving iptables" +/sbin/iptables-save + +echo "Configure Ops Agent log streams" +# TODO + +echo "Healthcheck API" +# TODO + + +SQUID_SERVER="127.0.0.1" +# Interface connected to Internet +INTERNET="ens4" +# Interface connected to LAN +LAN_IN="lo" +# Squid port +SQUID_PORT="3128" +# Clean old firewall +systemctl start libvirtd + + +echo "environment variables config" +echo "Configure Ops Agent log streams" +# TODO + +echo "Healthcheck API" +# TODO + diff --git a/gcp/config/squid.tpl b/gcp/config/squid.tpl new file mode 100644 index 0000000..07bf3dc --- /dev/null +++ b/gcp/config/squid.tpl @@ -0,0 +1,29 @@ +hosts_file /etc/hosts +acl all src 0.0.0.0/0.0.0.0 +acl localhost src 127.0.0.1/255.255.255.255 +acl to_localhost dst 127.0.0.0/8 +acl CONNECT method CONNECT + +# connection to http, https and ssl +acl Safe_ports port 80 +acl Safe_ports port 443 +acl SSL_ports port 22 + +acl src_whitelist srcdomain "/etc/squid/src_whitelist.txt" +acl dst_whitelist dstdomain "/etc/squid/dst_whitelist.txt" +acl lan src ${cidr} +acl my_phisical src 31.182.219.133 + +http_access allow src_whitelist +http_access allow dst_whitelist +http_access allow CONNECT +http_access deny CONNECT !SSL_ports +http_access deny !Safe_ports + +http_access allow localhost +http_access allow lan +http_access allow my_phisical +http_access deny all +http_reply_access allow all +http_port 3128 transparent +coredump_dir /var/spool/squid \ No newline at end of file diff --git a/gcp/config/startup.sh b/gcp/config/startup.sh deleted file mode 100644 index d89e0ff..0000000 --- a/gcp/config/startup.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -set -x # Debug -set -e # Strict exit codes - -echo "Applying system patches" -sudo apt-get update; - -sudo apt-get upgrade -y; - - -echo "Installing Ops Agent"; -curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh; -bash add-google-cloud-ops-agent-repo.sh --also-install ; - -echo "Installing wget"; -sudo apt-get install wget -y; - -echo "Installing MITM Proxy"; -if [[ $instance_name =~ ^packer.*] ]; -then -sudo wget https://snapshots.mitmproxy.org/8.1.1/mitmproxy-8.1.1-linux.tar.gz ; -tar -zxvf mitmproxy-8.1.1-linux.tar.gz; -rm mitmproxy-8.1.1-linux.tar.gz; -./mitmproxy --mode transparent --showhost; -fi - -echo "Apply MITM configuration"; -echo "enable forwarding mode"; -sudo sysctl -w net.ipv4.ip_forward=1; -sudo sysctl -w net.ipv6.conf.all.forwarding=1; - -echo "disable ICMP redirects"; -sudo sysctl -w net.ipv4.conf.all.send_redirects=0; - -echo "configure iptables"; - -sudo iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080; -sudo iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8080; -sudo ip6tables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080; -sudo ip6tables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8080; - -echo "start mitmproxy if not in packer"; -instance_name=$(curl http://metadata.google.internal/computeMetadata/v1/instance/id -H "Metadata-Flavor: Google") -if [[ ! $instance_name =~ ^packer.*] ]; then ./mitmproxy --mode transparent --showhost; fi - - -echo "Configure Ops Agent log streams"; -# TODO - -echo "Healthcheck API"; -# TODO diff --git a/gcp/config/whitelist.txt b/gcp/config/whitelist.txt new file mode 100644 index 0000000..0993fa0 --- /dev/null +++ b/gcp/config/whitelist.txt @@ -0,0 +1,2 @@ +metadata.google.internal +www.google.com \ No newline at end of file diff --git a/gcp/ilb.tf b/gcp/ilb.tf index 6e5ff42..b2027df 100644 --- a/gcp/ilb.tf +++ b/gcp/ilb.tf @@ -2,16 +2,14 @@ resource "google_compute_forwarding_rule" "egress_filter_forwarding_rule" { name = "egress-filter-forwarding-rule" backend_service = google_compute_region_backend_service.egress_filter_backend.id - region = var.region - ip_protocol = "TCP" load_balancing_scheme = "INTERNAL" all_ports = true allow_global_access = true - - network = var.vpc_name - subnetwork = var.subnet_name + project = var.project + network = var.vpc_name + subnetwork = var.subnet_name } # backend service @@ -39,6 +37,6 @@ resource "google_compute_region_health_check" "egress_filter" { unhealthy_threshold = 10 tcp_health_check { - port = 8080 + port = 80 } } \ No newline at end of file diff --git a/gcp/images.tf b/gcp/images.tf index 3a4e528..bfd9193 100644 --- a/gcp/images.tf +++ b/gcp/images.tf @@ -5,8 +5,8 @@ data "packer_version" "version" {} data "packer_files" "packer_config_dir" { directory = "${path.module}/packer" } -data "packer_files" "vm_config_files"{ - directory = "${path.module}/config" +data "packer_files" "vm_config_files" { + directory = "${path.module}/config" } @@ -36,5 +36,6 @@ output "packer_version" { } output "image_id" { - value = packer_image.solid_proxy_image.id + value = packer_image.solid_proxy_image.id } + diff --git a/gcp/main.tf b/gcp/main.tf index 1ae263b..dfbcfb8 100644 --- a/gcp/main.tf +++ b/gcp/main.tf @@ -1,3 +1,27 @@ + +data "google_compute_subnetwork" "subnet" { + name = var.subnet_name + region = var.region + project = var.project +} +data "template_file" "squid_config" { + template = file("${path.module}/config/squid.tpl") + vars = { + cidr = data.google_compute_subnetwork.subnet.ip_cidr_range + } +} + +data "template_file" "startup_script" { + template = file("${path.module}/config/config.tpl") + vars = { + squid_config = data.template_file.squid_config.rendered + whitelist = file("${path.module}/config/whitelist.txt") + } +} +output "startup_script" { + value = data.template_file.startup_script.rendered +} + resource "google_compute_instance_template" "egress_filter" { project = var.project name_prefix = "${local.prefix}egress-filter" @@ -14,7 +38,7 @@ resource "google_compute_instance_template" "egress_filter" { auto_delete = false - source_image = "packer-1663596114" + source_image = "ubuntu-1804-bionic-arm64-v20221201" # TODO KMS support # disk_encryption_key { # } @@ -22,7 +46,7 @@ resource "google_compute_instance_template" "egress_filter" { network_interface { network = var.vpc_name - subnetwork = var.subnet_name + subnetwork = data.google_compute_subnetwork.subnet.self_link } can_ip_forward = true @@ -32,9 +56,7 @@ resource "google_compute_instance_template" "egress_filter" { metadata = { # TODO - base image from packer or with startup script - startup-script = templatefile("${path.module}/config/startup.sh", { - - }) + startup-script = data.template_file.startup_script.rendered } # TODO @@ -60,13 +82,17 @@ resource "google_compute_region_instance_group_manager" "egress_filter" { name = "egress-filter-igm" base_instance_name = "egress-filter" - region = var.region + region = var.region - target_pools = [ google_compute_target_pool.egress_filter.id ] + target_pools = [google_compute_target_pool.egress_filter.id] version { instance_template = google_compute_instance_template.egress_filter.id } + named_port { + name = "http" + port = 80 + } } resource "google_compute_target_pool" "egress_filter" { @@ -81,7 +107,7 @@ resource "google_compute_region_autoscaler" "egress_filter" { target = google_compute_region_instance_group_manager.egress_filter.id autoscaling_policy { - mode = var.autoscaling_mode + mode = var.autoscaling_mode max_replicas = var.autoscaling_max_replicas min_replicas = var.autoscaling_min_replicas cooldown_period = var.autoscaling_cooldown_period @@ -91,3 +117,4 @@ resource "google_compute_region_autoscaler" "egress_filter" { } } } + diff --git a/gcp/packer/solid-proxy-e2-small.pkr.hcl b/gcp/packer/solid-proxy-e2-small.pkr.hcl index f6385b5..ca93ba4 100644 --- a/gcp/packer/solid-proxy-e2-small.pkr.hcl +++ b/gcp/packer/solid-proxy-e2-small.pkr.hcl @@ -1,18 +1,21 @@ -source "googlecompute" "mitm-proxy" { +source "googlecompute" "squid-proxy" { machine_type = "e2-standard-4" project_id = "solid-egress-filter" source_image_family = "ubuntu-1804-lts" ssh_username = "solid-egress-filter" - startup_script_file = "${path.root}/../config/startup.sh" + startup_script_file = "${path.module}/../config/startup.sh" + network = "projects/solid-egress-filter/global/networks/dev-vpc" + subnetwork = "projects/solid-egress-filter/regions/europe-west1/subnetworks/dev-subnet" zone = "europe-west1-b" } build { name = "solid-proxy-disk" - sources = ["source.googlecompute.mitm-proxy"] - # provisioner "file"{ - # source = "" - # destination ="/home/solid-egres-filter/mitm-config" - # } + sources = ["source.googlecompute.squid-proxy"] + + provisioner "file"{ + source = "" + destination ="/home/solid-egres-filter/squid-config" + } } diff --git a/gcp/terraform.tfvars b/gcp/terraform.tfvars new file mode 100644 index 0000000..01bbe9d --- /dev/null +++ b/gcp/terraform.tfvars @@ -0,0 +1,3 @@ +project = "solid-egress-filter" +region = "europe-west1" +prefix = "" diff --git a/gcp/variables.tf b/gcp/variables.tf index a87534c..e9e5288 100644 --- a/gcp/variables.tf +++ b/gcp/variables.tf @@ -18,6 +18,7 @@ variable "subnet_name" { default = "default" } + variable "prefix" { description = "Prefix for all resource names. Allows to deploy multiple instances in the same project." type = string @@ -49,21 +50,21 @@ variable "sa_name" { } variable "autoscaling_mode" { - type = string + type = string default = "ON" } variable "autoscaling_min_replicas" { - type = number + type = number default = 1 } variable "autoscaling_max_replicas" { - type = number + type = number default = 3 } variable "autoscaling_cooldown_period" { - type = number + type = number default = 300 }