From 44991c07ea398baaf94f964d5d42edd50c485f73 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Wed, 20 Jul 2022 18:45:06 -0700 Subject: [PATCH 1/4] ff.env_var_strict_mode: true --- data/meltano.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/data/meltano.yml b/data/meltano.yml index de307689..ff8aea96 100644 --- a/data/meltano.yml +++ b/data/meltano.yml @@ -2,6 +2,7 @@ version: 1 send_anonymous_usage_stats: true project_id: c15e971a-d318-4a9d-979b-1039ce5fd1b1 default_environment: userdev + include_paths: - ./environments/*.meltano.yml - ./extract/*.meltano.yml @@ -9,3 +10,8 @@ include_paths: - ./orchestrate/*.meltano.yml - ./transform/*.meltano.yml - ./utilities/*.meltano.yml + +# Feature Flags - https://docs.meltano.com/reference/settings#feature-flags + +# fail if any referenced env vars are blank +ff.env_var_strict_mode: true From 98e9187d90ece992566f985f4a3931b24c9ca065 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Wed, 20 Jul 2022 18:46:59 -0700 Subject: [PATCH 2/4] Update meltano.yml --- data/meltano.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/meltano.yml b/data/meltano.yml index ff8aea96..f7bd8e35 100644 --- a/data/meltano.yml +++ b/data/meltano.yml @@ -13,5 +13,5 @@ include_paths: # Feature Flags - https://docs.meltano.com/reference/settings#feature-flags -# fail if any referenced env vars are blank +# fail if any referenced env vars are missing ff.env_var_strict_mode: true From b46206233e0702691585e45a3c6b611dca422dc2 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Wed, 31 Aug 2022 14:22:03 -0700 Subject: [PATCH 3/4] fix spelling --- data/meltano.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/meltano.yml b/data/meltano.yml index c342ded7..0ce71ee0 100644 --- a/data/meltano.yml +++ b/data/meltano.yml @@ -14,4 +14,4 @@ include_paths: # Feature Flags - https://docs.meltano.com/reference/settings#feature-flags # fail if any referenced env vars are missing -ff.env_var_strict_mode: true +ff.strict_env_var_mode: true From b8b4d1b60c89ab849bc0978e698e424efb11362e Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Fri, 16 Dec 2022 13:30:33 -0500 Subject: [PATCH 4/4] cleanup how envs are used so strict mode will fire --- data/.env.template | 6 +++--- data/environments/userdev.meltano.yml | 1 - data/meltano.yml | 1 - data/transform/profiles/snowflake/profiles.yml | 10 +++++----- data/transform/transformers.meltano.yml | 1 + data/utilities/utilities.meltano.yml | 3 +++ 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/data/.env.template b/data/.env.template index bac0cdb2..dd663786 100644 --- a/data/.env.template +++ b/data/.env.template @@ -1,3 +1,6 @@ +# Developer Prefix +USER_PREFIX=PNADOLNY + # Google Analytics Tap TAP_GOOGLE_ANALYTICS_CLIENT_SECRETS="{\"type\": \"service_account\",\"project_id\": \"foo\",\"private_key_id\": \"bar\",\"private_key\": \"-----BEGIN PRIVATE KEY-----\\nfoo\\nbar\\n-----END PRIVATE KEY-----\\n\",\"client_email\": \"foo@bar.iam.gserviceaccount.com\",\"client_id\": \"foo\",\"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\"token_uri\": \"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/foobar.iam.gserviceaccount.com\"}" @@ -17,8 +20,5 @@ PERMISSION_BOT_ACCOUNT="****" # Snowflake MELTANO User Password SNOWFLAKE_PASSWORD="****" -# dbt Snowflake Password -DBT_SNOWFLAKE_PASSWORD="****" - # Slack Webhook TARGET_APPRISE_URIS=["https://hooks.slack.com/services/{}/{}/{}"] diff --git a/data/environments/userdev.meltano.yml b/data/environments/userdev.meltano.yml index 064ca83f..4932617a 100644 --- a/data/environments/userdev.meltano.yml +++ b/data/environments/userdev.meltano.yml @@ -75,7 +75,6 @@ environments: role: ${USER_PREFIX} warehouse: CORE env: - USER_PREFIX: PNADOLNY HUB_METRICS_S3_PATH: s3://devtest-meltano-bucket-01/hub_metrics/ SUPERSET_API_URL: http://localhost:8088 SUPERSET_USER: admin diff --git a/data/meltano.yml b/data/meltano.yml index 47317f77..3cf9d05a 100644 --- a/data/meltano.yml +++ b/data/meltano.yml @@ -12,6 +12,5 @@ include_paths: - ./utilities/*.meltano.yml # Feature Flags - https://docs.meltano.com/reference/settings#feature-flags - # fail if any referenced env vars are missing ff.strict_env_var_mode: true diff --git a/data/transform/profiles/snowflake/profiles.yml b/data/transform/profiles/snowflake/profiles.yml index c49492f4..12a98713 100644 --- a/data/transform/profiles/snowflake/profiles.yml +++ b/data/transform/profiles/snowflake/profiles.yml @@ -13,7 +13,7 @@ meltano: threads: 2 account: "{{ env_var('DBT_SNOWFLAKE_ACCOUNT') }}" user: "{{ env_var('DBT_SNOWFLAKE_USER') }}" - password: "{{ env_var('SNOWFLAKE_PASSWORD') }}" + password: "{{ env_var('DBT_SNOWFLAKE_PASSWORD') }}" role: "{{ env_var('DBT_SNOWFLAKE_ROLE') }}" warehouse: "{{ env_var('DBT_SNOWFLAKE_WAREHOUSE') }}" database: "{{ env_var('DBT_SNOWFLAKE_DATABASE') }}" @@ -23,7 +23,7 @@ meltano: threads: 2 account: "{{ env_var('DBT_SNOWFLAKE_ACCOUNT') }}" user: "{{ env_var('DBT_SNOWFLAKE_USER') }}" - password: "{{ env_var('SNOWFLAKE_PASSWORD') }}" + password: "{{ env_var('DBT_SNOWFLAKE_PASSWORD') }}" role: "{{ env_var('DBT_SNOWFLAKE_ROLE') }}" warehouse: "{{ env_var('DBT_SNOWFLAKE_WAREHOUSE') }}" database: "{{ env_var('DBT_SNOWFLAKE_DATABASE') }}" @@ -33,7 +33,7 @@ meltano: threads: 4 account: "{{ env_var('DBT_SNOWFLAKE_ACCOUNT') }}" user: "{{ env_var('DBT_SNOWFLAKE_USER') }}" - password: "{{ env_var('SNOWFLAKE_PASSWORD') }}" + password: "{{ env_var('DBT_SNOWFLAKE_PASSWORD') }}" role: "{{ env_var('DBT_SNOWFLAKE_ROLE') }}" warehouse: "{{ env_var('DBT_SNOWFLAKE_WAREHOUSE') }}" database: "{{ env_var('DBT_SNOWFLAKE_DATABASE') }}" @@ -43,7 +43,7 @@ meltano: threads: 6 account: "{{ env_var('DBT_SNOWFLAKE_ACCOUNT') }}" user: "{{ env_var('DBT_SNOWFLAKE_USER') }}" - password: "{{ env_var('SNOWFLAKE_PASSWORD') }}" + password: "{{ env_var('DBT_SNOWFLAKE_PASSWORD') }}" role: "{{ env_var('DBT_SNOWFLAKE_ROLE') }}" warehouse: "{{ env_var('DBT_SNOWFLAKE_WAREHOUSE') }}" database: "{{ env_var('DBT_SNOWFLAKE_DATABASE') }}" @@ -53,7 +53,7 @@ meltano: threads: 4 account: "{{ env_var('DBT_SNOWFLAKE_ACCOUNT') }}" user: "{{ env_var('DBT_SNOWFLAKE_USER') }}" - password: "{{ env_var('SNOWFLAKE_PASSWORD') }}" + password: "{{ env_var('DBT_SNOWFLAKE_PASSWORD') }}" role: "{{ env_var('DBT_SNOWFLAKE_ROLE') }}" warehouse: "{{ env_var('DBT_SNOWFLAKE_WAREHOUSE') }}" database: "{{ env_var('DBT_SNOWFLAKE_DATABASE') }}" diff --git a/data/transform/transformers.meltano.yml b/data/transform/transformers.meltano.yml index 79cb4462..76c51671 100644 --- a/data/transform/transformers.meltano.yml +++ b/data/transform/transformers.meltano.yml @@ -49,3 +49,4 @@ plugins: account: epa06486 database_raw: RAW database_prep: PREP + password: ${SNOWFLAKE_PASSWORD} diff --git a/data/utilities/utilities.meltano.yml b/data/utilities/utilities.meltano.yml index 401f632b..3c828a7c 100644 --- a/data/utilities/utilities.meltano.yml +++ b/data/utilities/utilities.meltano.yml @@ -7,6 +7,8 @@ plugins: settings: - name: user env: DBT_SNOWFLAKE_USER + - name: password + env: DBT_SNOWFLAKE_PASSWORD - name: role env: DBT_SNOWFLAKE_ROLE - name: database @@ -26,6 +28,7 @@ plugins: account: epa06486 role: TRANSFORMER warehouse: CORE + password: ${SNOWFLAKE_PASSWORD} - name: awscli namespace: awscli pip_url: awscli==1.21.7