From 35393568bf3c0a4a9e3ecf6f523f36da6da9444c Mon Sep 17 00:00:00 2001 From: Vinod Sugur Date: Wed, 16 Oct 2024 11:54:29 +0000 Subject: [PATCH 1/2] oracle adapter support Added Oracle adapter support --- macros/cross_db_utils.sql | 21 +++++++++++++++++---- macros/measures.sql | 35 ++++++++++++++++++++++++----------- macros/relation.sql | 6 ++++++ 3 files changed, 47 insertions(+), 15 deletions(-) diff --git a/macros/cross_db_utils.sql b/macros/cross_db_utils.sql index d9daa5d..13afb53 100644 --- a/macros/cross_db_utils.sql +++ b/macros/cross_db_utils.sql @@ -8,6 +8,10 @@ varchar {%- endmacro -%} +{%- macro oracle__type_string() -%} + varchar(1000) +{%- endmacro -%} + {%- macro bigquery__type_string() -%} string {%- endmacro -%} @@ -33,10 +37,6 @@ {% do return(is_numeric) %} {%- endmacro -%} -{%- macro athena__is_numeric_dtype(dtype) -%} - {% set is_numeric = "int" in dtype or "float" in dtype or "decimal" in dtype or "double" in dtype %} - {% do return(is_numeric) %} -{%- endmacro -%} {# is_logical_dtype ------------------------------------------------- #} @@ -80,6 +80,10 @@ {{ relation.information_schema() }} {%- endmacro -%} +{%- macro oracle__information_schema(relation) -%} + ALL_TAB_COLUMNS +{%- endmacro -%} + {%- macro bigquery__information_schema(relation) -%} {{ adapter.quote(relation.database) }}.{{ adapter.quote(relation.schema) }}.INFORMATION_SCHEMA {%- endmacro -%} @@ -100,6 +104,15 @@ order by ordinal_position asc {%- endmacro -%} +{%- macro oracle__select_from_information_schema_columns(relation) -%} + select + * + from {{ dbt_profiler.information_schema(relation) }} + where lower(owner) = lower('{{ relation.schema }}') + and lower(table_name) = lower('{{ relation.identifier }}') + order by column_id asc +{%- endmacro -%} + {%- macro redshift__select_from_information_schema_columns(relation) -%} select attr.attname::varchar as column_name, diff --git a/macros/measures.sql b/macros/measures.sql index fddf0da..44f0b8d 100644 --- a/macros/measures.sql +++ b/macros/measures.sql @@ -5,7 +5,7 @@ {%- endmacro -%} {%- macro default__measure_row_count(column_name, data_type) -%} -cast(count(*) as {{ dbt.type_int() }}) +cast(count(*) as {{ dbt.type_numeric() }}) {%- endmacro -%} @@ -19,6 +19,11 @@ cast(count(*) as {{ dbt.type_int() }}) sum(case when {{ adapter.quote(column_name) }} is null then 0 else 1 end) / cast(count(*) as {{ dbt.type_numeric() }}) {%- endmacro -%} +{%- macro oracle__measure_not_null_proportion(column_name, data_type) -%} +case when cast(count(*) as {{ dbt.type_numeric() }}) != 0 THEN +sum(case when {{ adapter.quote(column_name) }} is null then 0 else 1 end) / cast(count(*) as {{ dbt.type_numeric() }}) +ELSE 0 END +{%- endmacro -%} {# measure_distinct_proportion ------------------------------------------------- #} @@ -34,6 +39,16 @@ sum(case when {{ adapter.quote(column_name) }} is null then 0 else 1 end) / cast {%- endif -%} {%- endmacro -%} +{%- macro oracle__measure_distinct_proportion(column_name, data_type) -%} +{%- if not dbt_profiler.is_struct_dtype(data_type) -%} + CASE WHEN cast(count(*) as {{ dbt.type_numeric() }}) != 0 THEN + count(distinct {{ adapter.quote(column_name) }}) / cast(count(*) as {{ dbt.type_numeric() }}) + ELSE 0 END +{%- else -%} + cast(null as {{ dbt.type_numeric() }}) +{%- endif -%} +{%- endmacro -%} + {# measure_distinct_count ------------------------------------------------- #} {%- macro measure_distinct_count(column_name, data_type) -%} @@ -62,6 +77,14 @@ sum(case when {{ adapter.quote(column_name) }} is null then 0 else 1 end) / cast {%- endif -%} {%- endmacro -%} +{%- macro oracle__measure_is_unique(column_name, data_type) -%} +{%- if not dbt_profiler.is_struct_dtype(data_type) -%} + CASE WHEN count(distinct {{ adapter.quote(column_name) }}) = count(*) THEN 'Y' ELSE 'N' END +{%- else -%} + null +{%- endif -%} +{%- endmacro -%} + {%- macro sqlserver__measure_is_unique(column_name, data_type) -%} case when count(distinct {{ adapter.quote(column_name) }}) = count(*) then 1 else 0 end {%- endmacro -%} @@ -173,16 +196,6 @@ case when count(distinct {{ adapter.quote(column_name) }}) = count(*) then 1 els {%- endmacro -%} -{%- macro athena__measure_median(column_name, data_type, cte_name) -%} - -{%- if dbt_profiler.is_numeric_dtype(data_type) and not dbt_profiler.is_struct_dtype(data_type) -%} - approx_percentile( {{ adapter.quote(column_name) }}, 0.5) -{%- else -%} - cast(null as {{ dbt.type_numeric() }}) -{%- endif -%} - -{%- endmacro -%} - {%- macro sql_server__measure_median(column_name, data_type, cte_name) -%} {%- if dbt_profiler.is_numeric_dtype(data_type) and not dbt_profiler.is_struct_dtype(data_type) -%} diff --git a/macros/relation.sql b/macros/relation.sql index 7512b15..5ab5a37 100644 --- a/macros/relation.sql +++ b/macros/relation.sql @@ -45,4 +45,10 @@ {% do run_query("select top(0) * from " ~ relation ~ "") %} +{% endmacro %} + +{% macro oracle__assert_relation_exists(relation) %} + +{% do run_query("select * from " ~ relation ~ " where rownum < 0") %} + {% endmacro %} \ No newline at end of file From f1fc9d1af6c73589da5021bb100bc377e389f74f Mon Sep 17 00:00:00 2001 From: Vinod Sugur Date: Wed, 27 Nov 2024 13:01:27 +0000 Subject: [PATCH 2/2] Resolved open comments I have resolved open queries. --- README.md | 2 ++ macros/cross_db_utils.sql | 6 +++++- macros/measures.sql | 19 ++++++++++++++++--- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 761127d..0c697ab 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,8 @@ One of the advantages of the `doc` approach over the `meta` approach is that it ✅ Snowflake +✅ Oracle + ✅ SQL Server ❌ Apache Spark diff --git a/macros/cross_db_utils.sql b/macros/cross_db_utils.sql index 13afb53..21bc49b 100644 --- a/macros/cross_db_utils.sql +++ b/macros/cross_db_utils.sql @@ -37,6 +37,10 @@ {% do return(is_numeric) %} {%- endmacro -%} +{%- macro athena__is_numeric_dtype(dtype) -%} + {% set is_numeric = "int" in dtype or "float" in dtype or "decimal" in dtype or "double" in dtype %} + {% do return(is_numeric) %} +{%- endmacro -%} {# is_logical_dtype ------------------------------------------------- #} @@ -126,4 +130,4 @@ where lower(table_schema) = lower('{{ relation.schema }}') and lower(table_name) = lower('{{ relation.identifier }}') and attr.attnum > 0 -{%- endmacro -%} +{%- endmacro -%} \ No newline at end of file diff --git a/macros/measures.sql b/macros/measures.sql index 44f0b8d..adc6a42 100644 --- a/macros/measures.sql +++ b/macros/measures.sql @@ -5,9 +5,12 @@ {%- endmacro -%} {%- macro default__measure_row_count(column_name, data_type) -%} -cast(count(*) as {{ dbt.type_numeric() }}) +cast(count(*) as {{ dbt.type_bigint() }}) {%- endmacro -%} +{%- macro oracle__measure_row_count(column_name, data_type) -%} +cast(count(*) as {{ dbt.type_numeric() }}) +{%- endmacro -%} {# measure_not_null_proportion ------------------------------------------------- #} @@ -196,7 +199,17 @@ case when count(distinct {{ adapter.quote(column_name) }}) = count(*) then 1 els {%- endmacro -%} -{%- macro sql_server__measure_median(column_name, data_type, cte_name) -%} +{%- macro athena__measure_median(column_name, data_type, cte_name) -%} + +{%- if dbt_profiler.is_numeric_dtype(data_type) and not dbt_profiler.is_struct_dtype(data_type) -%} + approx_percentile( {{ adapter.quote(column_name) }}, 0.5) +{%- else -%} + cast(null as {{ dbt.type_numeric() }}) +{%- endif -%} + +{%- endmacro -%} + +{%- macro sqlserver__measure_median(column_name, data_type, cte_name) -%} {%- if dbt_profiler.is_numeric_dtype(data_type) and not dbt_profiler.is_struct_dtype(data_type) -%} percentile_cont({{ adapter.quote(column_name) }}, 0.5) over () @@ -259,4 +272,4 @@ case when count(distinct {{ adapter.quote(column_name) }}) = count(*) then 1 els cast(null as {{ dbt.type_numeric() }}) {%- endif -%} -{%- endmacro -%} +{%- endmacro -%} \ No newline at end of file