From 9cdcb7d80a088a0a1c92f187db69a9b802cd5676 Mon Sep 17 00:00:00 2001 From: laurentgrangeau Date: Mon, 22 Sep 2025 10:43:08 +0200 Subject: [PATCH 1/3] feat: placeholder for README --- .../base/use-cases/inference-ref-arch/examples/retail/README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 platforms/gke/base/use-cases/inference-ref-arch/examples/retail/README.md diff --git a/platforms/gke/base/use-cases/inference-ref-arch/examples/retail/README.md b/platforms/gke/base/use-cases/inference-ref-arch/examples/retail/README.md new file mode 100644 index 000000000..e45f84908 --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/examples/retail/README.md @@ -0,0 +1 @@ +# Retail use case based on the ComfyUI reference implementation \ No newline at end of file From dd60712dc2884add353d43e7a2101d1ef17ad1dd Mon Sep 17 00:00:00 2001 From: laurentgrangeau Date: Mon, 22 Sep 2025 18:02:48 +0200 Subject: [PATCH 2/3] feat: add retail dictionary --- .github/workflows/dictionary/retail.txt | 1 + cspell.json | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 .github/workflows/dictionary/retail.txt diff --git a/.github/workflows/dictionary/retail.txt b/.github/workflows/dictionary/retail.txt new file mode 100644 index 000000000..f1733209f --- /dev/null +++ b/.github/workflows/dictionary/retail.txt @@ -0,0 +1 @@ +workflowexecutions diff --git a/cspell.json b/cspell.json index 8870d2c84..5e91ac775 100644 --- a/cspell.json +++ b/cspell.json @@ -58,6 +58,10 @@ "name": "ray", "path": ".github/workflows/dictionary/ray.txt" }, + { + "name": "retail", + "path": ".github/workflows/dictionary/retail.txt" + }, { "name": "shell", "path": ".github/workflows/dictionary/shell.txt" From ca1b0db0190861ccd41f34f5be1c64233f0abee8 Mon Sep 17 00:00:00 2001 From: laurentgrangeau Date: Mon, 22 Sep 2025 18:15:47 +0200 Subject: [PATCH 3/3] feat: add retail use case --- .../_shared_config/retail.auto.tfvars | 14 ++++ .../_shared_config/retail_variables.tf | 17 +++++ .../terraform/retail/_cluster.auto.tfvars | 1 + .../terraform/retail/_cluster_variables.tf | 1 + .../terraform/retail/_networking.auto.tfvars | 1 + .../terraform/retail/_networking_variables.tf | 1 + .../terraform/retail/_platform.auto.tfvars | 1 + .../terraform/retail/_platform_variables.tf | 1 + .../terraform/retail/_retail.auto.tfvars | 1 + .../terraform/retail/_retail_variables.tf | 1 + .../terraform/retail/dataflow.tf | 36 +++++++++ .../terraform/retail/firewall.tf | 43 +++++++++++ .../terraform/retail/project.tf | 52 +++++++++++++ .../terraform/retail/pubsub.tf | 74 +++++++++++++++++++ .../terraform/retail/storage.tf | 47 ++++++++++++ .../terraform/retail/versions.tf | 28 +++++++ 16 files changed, 319 insertions(+) create mode 100644 platforms/gke/base/use-cases/inference-ref-arch/terraform/_shared_config/retail.auto.tfvars create mode 100644 platforms/gke/base/use-cases/inference-ref-arch/terraform/_shared_config/retail_variables.tf create mode 120000 platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_cluster.auto.tfvars create mode 120000 platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_cluster_variables.tf create mode 120000 platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_networking.auto.tfvars create mode 120000 platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_networking_variables.tf create mode 120000 platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_platform.auto.tfvars create mode 120000 platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_platform_variables.tf create mode 120000 platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_retail.auto.tfvars create mode 120000 platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_retail_variables.tf create mode 100644 platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/dataflow.tf create mode 100644 platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/firewall.tf create mode 100644 platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/project.tf create mode 100644 platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/pubsub.tf create mode 100644 platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/storage.tf create mode 100644 platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/versions.tf diff --git a/platforms/gke/base/use-cases/inference-ref-arch/terraform/_shared_config/retail.auto.tfvars b/platforms/gke/base/use-cases/inference-ref-arch/terraform/_shared_config/retail.auto.tfvars new file mode 100644 index 000000000..114b92c3a --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/terraform/_shared_config/retail.auto.tfvars @@ -0,0 +1,14 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/platforms/gke/base/use-cases/inference-ref-arch/terraform/_shared_config/retail_variables.tf b/platforms/gke/base/use-cases/inference-ref-arch/terraform/_shared_config/retail_variables.tf new file mode 100644 index 000000000..43819cbae --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/terraform/_shared_config/retail_variables.tf @@ -0,0 +1,17 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + dataflow_job_name = "${local.unique_identifier_prefix}-retail-job" +} \ No newline at end of file diff --git a/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_cluster.auto.tfvars b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_cluster.auto.tfvars new file mode 120000 index 000000000..98a694db9 --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_cluster.auto.tfvars @@ -0,0 +1 @@ +../../../../_shared_config/cluster.auto.tfvars \ No newline at end of file diff --git a/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_cluster_variables.tf b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_cluster_variables.tf new file mode 120000 index 000000000..00625515b --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_cluster_variables.tf @@ -0,0 +1 @@ +../../../../_shared_config/cluster_variables.tf \ No newline at end of file diff --git a/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_networking.auto.tfvars b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_networking.auto.tfvars new file mode 120000 index 000000000..9cbd92baf --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_networking.auto.tfvars @@ -0,0 +1 @@ +../../../../_shared_config/networking.auto.tfvars \ No newline at end of file diff --git a/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_networking_variables.tf b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_networking_variables.tf new file mode 120000 index 000000000..1e170e71d --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_networking_variables.tf @@ -0,0 +1 @@ +../../../../_shared_config/networking_variables.tf \ No newline at end of file diff --git a/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_platform.auto.tfvars b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_platform.auto.tfvars new file mode 120000 index 000000000..125a652cf --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_platform.auto.tfvars @@ -0,0 +1 @@ +../../../../_shared_config/platform.auto.tfvars \ No newline at end of file diff --git a/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_platform_variables.tf b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_platform_variables.tf new file mode 120000 index 000000000..486b3eaef --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_platform_variables.tf @@ -0,0 +1 @@ +../../../../_shared_config/platform_variables.tf \ No newline at end of file diff --git a/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_retail.auto.tfvars b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_retail.auto.tfvars new file mode 120000 index 000000000..a3dd69248 --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_retail.auto.tfvars @@ -0,0 +1 @@ +../_shared_config/retail.auto.tfvars \ No newline at end of file diff --git a/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_retail_variables.tf b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_retail_variables.tf new file mode 120000 index 000000000..60dfb99ad --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/_retail_variables.tf @@ -0,0 +1 @@ +../_shared_config/retail_variables.tf \ No newline at end of file diff --git a/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/dataflow.tf b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/dataflow.tf new file mode 100644 index 000000000..7111bc40b --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/dataflow.tf @@ -0,0 +1,36 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + template_gcs_path = "gs://${google_storage_bucket.dataflow_staging_bucket.name}/templates/aggregate.template" + df_tmp_path = "gs://${google_storage_bucket.dataflow_staging_bucket.name}/tmp" +} + +resource "google_dataflow_job" "batching_dataflow" { + name = local.dataflow_job_name + region = local.cluster_region + project = google_project_service.dataflow_googleapis_com.project + + template_gcs_path = local.template_gcs_path + temp_gcs_location = local.df_tmp_path + + enable_streaming_engine = true + on_delete = "cancel" + + depends_on = [ + google_project_iam_binding.pubsub_subscriber_role, + google_project_iam_binding.pubsub_publisher_role, + google_project_iam_binding.pubsub_viewer_role, + ] +} \ No newline at end of file diff --git a/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/firewall.tf b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/firewall.tf new file mode 100644 index 000000000..c3dd98951 --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/firewall.tf @@ -0,0 +1,43 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "google_compute_firewall" "dataflow_ingress" { + name = "dataflow-ingress" + network = local.network_cluster_network_name + direction = "INGRESS" + project = data.google_project.cluster.project_id + + allow { + protocol = "tcp" + ports = ["12345-12346"] + } + + source_tags = ["dataflow"] + target_tags = ["dataflow"] +} + +resource "google_compute_firewall" "dataflow_egress" { + name = "dataflow-egress" + network = local.network_cluster_network_name + direction = "EGRESS" + project = data.google_project.cluster.project_id + + allow { + protocol = "tcp" + ports = ["12345-12346"] + } + + destination_ranges = ["10.0.0.0/8"] + target_tags = ["dataflow"] +} diff --git a/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/project.tf b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/project.tf new file mode 100644 index 000000000..f7dbb4c3f --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/project.tf @@ -0,0 +1,52 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +data "google_project" "cluster" { + project_id = local.cluster_project_id +} + +resource "google_project_service" "dataflow_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.cluster.project_id + service = "dataflow.googleapis.com" +} + +resource "google_project_service" "pubsub_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.cluster.project_id + service = "pubsub.googleapis.com" +} + +resource "google_project_service" "alloydb_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.cluster.project_id + service = "alloydb.googleapis.com" +} + +resource "google_project_service" "workflows_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.cluster.project_id + service = "workflows.googleapis.com" +} + +resource "google_project_service" "workflowexecutions_googleapis_com" { + disable_dependent_services = false + disable_on_destroy = false + project = data.google_project.cluster.project_id + service = "workflowexecutions.googleapis.com" +} diff --git a/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/pubsub.tf b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/pubsub.tf new file mode 100644 index 000000000..8ad5de625 --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/pubsub.tf @@ -0,0 +1,74 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + incoming_pubsub_topic_name = "${local.unique_identifier_prefix_underscore}_incoming_retail_event" +} + +resource "google_pubsub_topic" "incoming_pubsub_topic" { + name = local.incoming_pubsub_topic_name + project = google_project_service.pubsub_googleapis_com.project +} + +resource "google_pubsub_topic_iam_binding" "publisher_binding" { + project = google_project_service.pubsub_googleapis_com.project + topic = google_pubsub_topic.incoming_pubsub_topic.name + role = "roles/pubsub.publisher" + members = [ + local.publisher_sa, + ] +} + +resource "google_pubsub_topic" "incoming_pubsub_dlq" { + name = "incoming_images_dlq" + project = google_project_service.pubsub_googleapis_com.project +} + +resource "google_pubsub_subscription" "incoming_pubsub_subscription" { + name = "incoming_subscription_ap" + topic = google_pubsub_topic.incoming_pubsub_topic.name + project = google_project_service.pubsub_googleapis_com.project +} + +resource "google_pubsub_topic" "workflow1_pubsub_topic" { + name = "workflow1_pubsub_topic" + project = google_project_service.pubsub_googleapis_com.project +} + +resource "google_pubsub_subscription" "incoming_dlq_subscription" { + name = "${google_pubsub_subscription.incoming_pubsub_subscription.name}_dlq" + topic = google_pubsub_topic.incoming_pubsub_dlq.id + project = google_project_service.pubsub_googleapis_com.project + expiration_policy { + ttl = "" + } +} + +resource "google_pubsub_topic_iam_binding" "incoming_dlq_grant_publish" { + topic = google_pubsub_topic.incoming_pubsub_dlq.id + role = "roles/pubsub.publisher" + members = [ + local.pubsub_sa_name, + ] + project = google_project_service.pubsub_googleapis_com.project +} + +resource "google_pubsub_subscription_iam_binding" "incoming_grant_subscribe" { + subscription = google_pubsub_subscription.incoming_pubsub_subscription.name + role = "roles/pubsub.subscriber" + members = [ + local.pubsub_sa_name, + ] + project = google_project_service.pubsub_googleapis_com.project +} diff --git a/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/storage.tf b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/storage.tf new file mode 100644 index 000000000..58d7922ed --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/storage.tf @@ -0,0 +1,47 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + dataflow_staging_bucket = "${local.unique_identifier_prefix}-dataflow-staging" + batches_bucket = "${local.unique_identifier_prefix}-batches" +} + +resource "google_storage_bucket" "batches_bucket" { + name = local.batches_bucket + location = local.cluster_region + project = data.google_project.cluster.project_id + public_access_prevention = "enforced" + uniform_bucket_level_access = true + + lifecycle_rule { + action { + type = "Delete" + } + condition { + age = 7 + } + } +} + +resource "google_storage_bucket" "dataflow_staging_bucket" { + name = local.dataflow_staging_bucket + location = local.cluster_region + project = data.google_project.cluster.project_id + public_access_prevention = "enforced" + uniform_bucket_level_access = true + + hierarchical_namespace { + enabled = true + } +} diff --git a/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/versions.tf b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/versions.tf new file mode 100644 index 000000000..ef83f58ac --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/terraform/retail/versions.tf @@ -0,0 +1,28 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.5.7" + + required_providers { + google = { + source = "hashicorp/google" + version = "6.49.2" + } + } + + provider_meta "google" { + module_name = "cloud-solutions/retail_deploy-v1" + } +}