Skip to content

Commit 2718cda

Browse files
authored
[CT-1970]: convert custom_schema tests to functional (#454)
* convert custom_schema tests to functional * Replace bigserial with bigint * nit newline * import ordering * Pr comments cleanup * scope of setUp to function * Simplify seeds addition * Nits
1 parent 33fdb84 commit 2718cda

File tree

13 files changed

+217
-199
lines changed

13 files changed

+217
-199
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
seed_csv = """
2+
id,first_name,last_name,email,gender,ip_address
3+
1,Jack,Hunter,[email protected],Male,59.80.20.168
4+
2,Kathryn,Walker,[email protected],Female,194.121.179.35
5+
3,Gerald,Ryan,[email protected],Male,11.3.212.243
6+
""".lstrip()
7+
8+
seed_agg_csv = """
9+
last_name,count
10+
Hunter,2
11+
Walker,2
12+
Ryan,2
13+
""".lstrip()
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
import pytest
2+
import os
3+
from dbt.tests.util import (
4+
check_relations_equal,
5+
check_table_does_exist,
6+
run_dbt
7+
)
8+
from tests.functional.adapter.custom_schema_tests.seeds import (
9+
seed_agg_csv,
10+
seed_csv
11+
)
12+
13+
_VIEW_1_SQL = """
14+
select * from {{ ref('seed') }}
15+
""".lstrip()
16+
17+
_VIEW_2_SQL = """
18+
{{ config(database='alt') }}
19+
select * from {{ ref('view_1') }}
20+
""".lstrip()
21+
22+
_VIEW_3_SQL = """
23+
{{ config(database='alt', materialized='table') }}
24+
25+
26+
with v1 as (
27+
28+
select * from {{ ref('view_1') }}
29+
30+
),
31+
32+
v2 as (
33+
34+
select * from {{ ref('view_2') }}
35+
36+
),
37+
38+
combined as (
39+
40+
select last_name from v1
41+
union all
42+
select last_name from v2
43+
44+
)
45+
46+
select
47+
last_name,
48+
count(*) as count
49+
50+
from combined
51+
group by 1
52+
""".lstrip()
53+
54+
_CUSTOM_DB_SQL = """
55+
{% macro generate_database_name(database_name, node) %}
56+
{% if database_name == 'alt' %}
57+
{{ env_var('SNOWFLAKE_TEST_ALT_DATABASE') }}
58+
{% elif database_name %}
59+
{{ database_name }}
60+
{% else %}
61+
{{ target.database }}
62+
{% endif %}
63+
{% endmacro %}
64+
""".lstrip()
65+
66+
ALT_DATABASE = os.getenv("SNOWFLAKE_TEST_ALT_DATABASE")
67+
68+
class TestOverrideDatabase:
69+
@pytest.fixture(scope="class")
70+
def macros(self):
71+
return {
72+
"custom_db.sql": _CUSTOM_DB_SQL,
73+
}
74+
75+
@pytest.fixture(scope="class")
76+
def seeds(self):
77+
return {
78+
"seed.csv" : seed_csv,
79+
"agg.csv": seed_agg_csv
80+
}
81+
82+
@pytest.fixture(scope="class")
83+
def models(self):
84+
return {
85+
"view_1.sql": _VIEW_1_SQL,
86+
"view_2.sql": _VIEW_2_SQL,
87+
"view_3.sql": _VIEW_3_SQL,
88+
}
89+
90+
def test_snowflake_override_generate_db_name(self, project):
91+
seed_results = run_dbt(["seed", "--full-refresh"])
92+
assert len(seed_results) == 2
93+
94+
db_with_schema = f"{project.database}.{project.test_schema}"
95+
alt_db_with_schema = f"{ALT_DATABASE}.{project.test_schema}"
96+
seed_table = "SEED"
97+
agg_table = "AGG"
98+
view_1 = "VIEW_1"
99+
view_2 = "VIEW_2"
100+
view_3 = "VIEW_3"
101+
102+
check_table_does_exist(project.adapter, f"{db_with_schema}.{seed_table}")
103+
check_table_does_exist(project.adapter, f"{db_with_schema}.{agg_table}")
104+
105+
results = run_dbt()
106+
assert len(results) == 3
107+
108+
check_table_does_exist(project.adapter, f"{db_with_schema}.{view_1}")
109+
check_table_does_exist(project.adapter, f"{alt_db_with_schema}.{view_2}")
110+
check_table_does_exist(project.adapter, f"{alt_db_with_schema}.{view_3}")
111+
112+
# not overridden
113+
check_relations_equal(project.adapter, [f"{db_with_schema}.{seed_table}", f"{db_with_schema}.{view_1}"])
114+
115+
# overridden
116+
check_relations_equal(project.adapter, [f"{db_with_schema}.{seed_table}", f"{alt_db_with_schema}.{view_2}"])
117+
check_relations_equal(project.adapter, [f"{db_with_schema}.{agg_table}", f"{alt_db_with_schema}.{view_3}"])
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
import pytest
2+
from dbt.tests.util import (
3+
check_relations_equal,
4+
run_dbt
5+
)
6+
from tests.functional.adapter.custom_schema_tests.seeds import (
7+
seed_agg_csv,
8+
seed_csv
9+
)
10+
11+
_VIEW_1_SQL = """
12+
select * from {{ ref('seed') }}
13+
""".lstrip()
14+
15+
_VIEW_2_SQL = """
16+
{{ config(schema='custom') }}
17+
18+
select * from {{ ref('view_1') }}
19+
""".lstrip()
20+
21+
_VIEW_3_SQL = """
22+
{{ config(schema='test', materialized='table') }}
23+
24+
25+
with v1 as (
26+
27+
select * from {{ ref('view_1') }}
28+
29+
),
30+
31+
v2 as (
32+
33+
select * from {{ ref('view_2') }}
34+
35+
),
36+
37+
combined as (
38+
39+
select last_name from v1
40+
union all
41+
select last_name from v2
42+
43+
)
44+
45+
select
46+
last_name,
47+
count(*) as count
48+
49+
from combined
50+
group by 1
51+
""".lstrip()
52+
53+
54+
class TestCustomProjectSchemaWithPrefix:
55+
@pytest.fixture(scope="class")
56+
def seeds(self):
57+
return {
58+
"seed.csv": seed_csv,
59+
"agg.csv": seed_agg_csv
60+
}
61+
62+
@pytest.fixture(scope="class")
63+
def models(self):
64+
return {
65+
"view_1.sql": _VIEW_1_SQL,
66+
"view_2.sql": _VIEW_2_SQL,
67+
"view_3.sql": _VIEW_3_SQL
68+
}
69+
70+
@pytest.fixture(scope="class")
71+
def project_config_update(self):
72+
return {
73+
"models": {
74+
"schema": "dbt_test"
75+
}
76+
}
77+
78+
def test__snowflake__custom_schema_with_prefix(self, project):
79+
seed_results = run_dbt(["seed"])
80+
assert len(seed_results) == 2
81+
results = run_dbt()
82+
assert len(results) == 3
83+
84+
db_with_schema = f"{project.database}.{project.test_schema}"
85+
check_relations_equal(project.adapter, [f"{db_with_schema}.SEED", f"{db_with_schema}_DBT_TEST.VIEW_1"])
86+
check_relations_equal(project.adapter, [f"{db_with_schema}.SEED", f"{db_with_schema}_CUSTOM.VIEW_2"])
87+
check_relations_equal(project.adapter, [f"{db_with_schema}.AGG", f"{db_with_schema}_TEST.VIEW_3"])

tests/integration/custom_schema_test/custom-db-macros/custom_db.sql

-10
This file was deleted.

tests/integration/custom_schema_test/db-models/view_1.sql

-3
This file was deleted.

tests/integration/custom_schema_test/db-models/view_2.sql

-2
This file was deleted.

tests/integration/custom_schema_test/db-models/view_3.sql

-30
This file was deleted.

tests/integration/custom_schema_test/models/view_1.sql

-3
This file was deleted.

tests/integration/custom_schema_test/models/view_2.sql

-4
This file was deleted.

tests/integration/custom_schema_test/models/view_3.sql

-30
This file was deleted.

tests/integration/custom_schema_test/seed.sql

-25
This file was deleted.

tests/integration/custom_schema_test/test_custom_database.py

-39
This file was deleted.

0 commit comments

Comments
 (0)