Skip to content

Commit 8859a27

Browse files
Samk13Samk13
committed
config: granular env-based solution for connection strings
Add logic to build connection urls from env vars if available needed for helm charts security best practices. includes: * Build db uri * Build redis url * Build mq url Partially closes: inveniosoftware/helm-invenio#112
1 parent e3bc51c commit 8859a27

File tree

2 files changed

+289
-0
lines changed

2 files changed

+289
-0
lines changed

‎invenio_config/env.py

+100
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#
33
# This file is part of Invenio.
44
# Copyright (C) 2015-2018 CERN.
5+
# Copyright (C) 2024 KTH Royal Institute of Technology.
56
#
67
# Invenio is free software; you can redistribute it and/or modify it
78
# under the terms of the MIT License; see LICENSE file for more details.
@@ -45,3 +46,102 @@ def init_app(self, app):
4546

4647
# Set value
4748
app.config[varname] = value
49+
50+
51+
def _get_env_var(prefix, keys):
52+
"""Retrieve environment variables with a given prefix."""
53+
return {k: os.environ.get(f"{prefix}_{k.upper()}") for k in keys}
54+
55+
56+
def build_db_uri():
57+
"""
58+
Build database URI from environment variables or use default.
59+
60+
Priority order:
61+
1. INVENIO_SQLALCHEMY_DATABASE_URI
62+
2. SQLALCHEMY_DATABASE_URI
63+
3. INVENIO_DB_* specific environment variables
64+
4. Default URI
65+
66+
Note: For option 3, to assert that the INVENIO_DB_* settings take effect,
67+
you need to set SQLALCHEMY_DATABASE_URI="" in your environment.
68+
"""
69+
default_uri = "postgresql+psycopg2://invenio-app-rdm:invenio-app-rdm@localhost/invenio-app-rdm"
70+
71+
uri = os.environ.get("INVENIO_SQLALCHEMY_DATABASE_URI") or os.environ.get(
72+
"SQLALCHEMY_DATABASE_URI"
73+
)
74+
if uri:
75+
return uri
76+
77+
db_params = _get_env_var(
78+
"INVENIO_DB", ["user", "password", "host", "port", "name", "protocol"]
79+
)
80+
if all(db_params.values()):
81+
uri = f"{db_params['protocol']}://{db_params['user']}:{db_params['password']}@{db_params['host']}:{db_params['port']}/{db_params['name']}"
82+
return uri
83+
84+
return default_uri
85+
86+
87+
def build_broker_url():
88+
"""
89+
Build broker URL from environment variables or use default.
90+
91+
Priority order:
92+
1. INVENIO_BROKER_URL
93+
2. BROKER_URL
94+
3. INVENIO_AMQP_BROKER_* specific environment variables
95+
4. Default URL
96+
Note: see: https://docs.celeryq.dev/en/stable/userguide/configuration.html#new-lowercase-settings
97+
"""
98+
default_url = "amqp://guest:guest@localhost:5672/"
99+
100+
broker_url = os.environ.get("INVENIO_BROKER_URL") or os.environ.get("BROKER_URL")
101+
if broker_url:
102+
return broker_url
103+
104+
broker_params = _get_env_var(
105+
"INVENIO_AMQP_BROKER", ["user", "password", "host", "port", "protocol"]
106+
)
107+
if all(broker_params.values()):
108+
vhost = f"{os.environ.get('INVENIO_AMQP_BROKER_VHOST').removeprefix('/')}"
109+
broker_url = f"{broker_params['protocol']}://{broker_params['user']}:{broker_params['password']}@{broker_params['host']}:{broker_params['port']}/{vhost}"
110+
return broker_url
111+
return default_url
112+
113+
114+
def build_redis_url(db=None):
115+
"""
116+
Build Redis URL from environment variables or use default.
117+
118+
Priority order:
119+
1. INVENIO_CACHE_REDIS_URL
120+
2. CACHE_REDIS_URL
121+
3. INVENIO_KV_CACHE_* specific environment variables
122+
4. Default URL
123+
"""
124+
db = db if db is not None else 0
125+
default_url = f"redis://localhost:6379/{db}"
126+
127+
cache_url = os.environ.get("INVENIO_CACHE_REDIS_URL") or os.environ.get(
128+
"CACHE_REDIS_URL"
129+
)
130+
if cache_url and cache_url.startswith(("redis://", "rediss://", "unix://")):
131+
return cache_url
132+
133+
redis_params = _get_env_var(
134+
"INVENIO_KV_CACHE", ["host", "port", "password", "protocol"]
135+
)
136+
137+
if redis_params["host"] and redis_params["port"]:
138+
protocol = redis_params.get("protocol", "redis")
139+
password = (
140+
f":{redis_params['password']}@" if redis_params.get("password") else ""
141+
)
142+
cache_url = (
143+
f"{protocol}://{password}{redis_params['host']}:{redis_params['port']}/{db}"
144+
)
145+
return cache_url
146+
147+
return default_url

‎tests/test_invenio_config.py

+189
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#
33
# This file is part of Invenio.
44
# Copyright (C) 2015-2018 CERN.
5+
# Copyright (C) 2024 KTH Royal Institute of Technology.
56
#
67
# Invenio is free software; you can redistribute it and/or modify it
78
# under the terms of the MIT License; see LICENSE file for more details.
@@ -16,6 +17,7 @@
1617
import warnings
1718
from os.path import join
1819

20+
import pytest
1921
from flask import Flask
2022
from mock import patch
2123
from pkg_resources import EntryPoint
@@ -29,6 +31,7 @@
2931
create_config_loader,
3032
)
3133
from invenio_config.default import ALLOWED_HTML_ATTRS, ALLOWED_HTML_TAGS
34+
from invenio_config.env import build_broker_url, build_db_uri, build_redis_url
3235

3336

3437
class ConfigEP(EntryPoint):
@@ -231,3 +234,189 @@ class Config(object):
231234
assert app.config["ENV"] == "env"
232235
finally:
233236
shutil.rmtree(tmppath)
237+
238+
239+
def set_env_vars(monkeypatch, env_vars):
240+
"""Helper function to set environment variables."""
241+
for key in env_vars:
242+
monkeypatch.delenv(key, raising=False)
243+
for key, value in env_vars.items():
244+
monkeypatch.setenv(key, value)
245+
246+
247+
@pytest.mark.parametrize(
248+
"env_vars, expected_uri",
249+
[
250+
(
251+
{
252+
"INVENIO_DB_USER": "testuser",
253+
"INVENIO_DB_PASSWORD": "testpassword",
254+
"INVENIO_DB_HOST": "testhost",
255+
"INVENIO_DB_PORT": "5432",
256+
"INVENIO_DB_NAME": "testdb",
257+
"INVENIO_DB_PROTOCOL": "postgresql+psycopg2",
258+
},
259+
"postgresql+psycopg2://testuser:testpassword@testhost:5432/testdb",
260+
),
261+
(
262+
{
263+
"INVENIO_SQLALCHEMY_DATABASE_URI": "postgresql+psycopg2://testuser:testpassword@testhost:5432/testdb"
264+
},
265+
"postgresql+psycopg2://testuser:testpassword@testhost:5432/testdb",
266+
),
267+
(
268+
{
269+
"SQLALCHEMY_DATABASE_URI": "postgresql+psycopg2://testuser:testpassword@testhost:5432/testdb"
270+
},
271+
"postgresql+psycopg2://testuser:testpassword@testhost:5432/testdb",
272+
),
273+
(
274+
{},
275+
"postgresql+psycopg2://invenio-app-rdm:invenio-app-rdm@localhost/invenio-app-rdm",
276+
),
277+
],
278+
)
279+
def test_build_db_uri(monkeypatch, env_vars, expected_uri):
280+
"""Test building database URI."""
281+
set_env_vars(monkeypatch, env_vars)
282+
assert build_db_uri() == expected_uri
283+
284+
285+
@pytest.mark.parametrize(
286+
"env_vars, expected_url",
287+
[
288+
(
289+
{
290+
"INVENIO_AMQP_BROKER_USER": "testuser",
291+
"INVENIO_AMQP_BROKER_PASSWORD": "testpassword",
292+
"INVENIO_AMQP_BROKER_HOST": "testhost",
293+
"INVENIO_AMQP_BROKER_PORT": "5672",
294+
"INVENIO_AMQP_BROKER_PROTOCOL": "amqp",
295+
"INVENIO_AMQP_BROKER_VHOST": "/testvhost",
296+
},
297+
"amqp://testuser:testpassword@testhost:5672/testvhost",
298+
),
299+
(
300+
{
301+
"INVENIO_AMQP_BROKER_USER": "testuser",
302+
"INVENIO_AMQP_BROKER_PASSWORD": "testpassword",
303+
"INVENIO_AMQP_BROKER_HOST": "testhost",
304+
"INVENIO_AMQP_BROKER_PORT": "5672",
305+
"INVENIO_AMQP_BROKER_PROTOCOL": "amqp",
306+
"INVENIO_AMQP_BROKER_VHOST": "testvhost",
307+
},
308+
"amqp://testuser:testpassword@testhost:5672/testvhost",
309+
),
310+
(
311+
{
312+
"INVENIO_AMQP_BROKER_USER": "testuser",
313+
"INVENIO_AMQP_BROKER_PASSWORD": "testpassword",
314+
"INVENIO_AMQP_BROKER_HOST": "testhost",
315+
"INVENIO_AMQP_BROKER_PORT": "5672",
316+
"INVENIO_AMQP_BROKER_PROTOCOL": "amqp",
317+
"INVENIO_AMQP_BROKER_VHOST": "",
318+
},
319+
"amqp://testuser:testpassword@testhost:5672/",
320+
),
321+
(
322+
{
323+
"INVENIO_AMQP_BROKER_USER": "testuser",
324+
"INVENIO_AMQP_BROKER_PASSWORD": "testpassword",
325+
"INVENIO_AMQP_BROKER_HOST": "testhost",
326+
"INVENIO_AMQP_BROKER_PORT": "5672",
327+
"INVENIO_AMQP_BROKER_PROTOCOL": "amqp",
328+
},
329+
"amqp://testuser:testpassword@testhost:5672/",
330+
),
331+
],
332+
)
333+
def test_build_broker_url_with_vhost(monkeypatch, env_vars, expected_url):
334+
"""Test building broker URL with vhost."""
335+
set_env_vars(monkeypatch, env_vars)
336+
assert build_broker_url() == expected_url
337+
338+
339+
@pytest.mark.parametrize(
340+
"env_vars, expected_url",
341+
[
342+
(
343+
{
344+
"INVENIO_AMQP_BROKER_USER": "testuser",
345+
"INVENIO_AMQP_BROKER_PASSWORD": "testpassword",
346+
"INVENIO_AMQP_BROKER_HOST": "testhost",
347+
"INVENIO_AMQP_BROKER_PORT": "5672",
348+
"INVENIO_AMQP_BROKER_PROTOCOL": "amqp",
349+
"INVENIO_AMQP_BROKER_VHOST": "testvhost",
350+
},
351+
"amqp://testuser:testpassword@testhost:5672/testvhost",
352+
),
353+
(
354+
{
355+
"INVENIO_AMQP_BROKER_USER": "testuser",
356+
"INVENIO_AMQP_BROKER_PASSWORD": "testpassword",
357+
"INVENIO_AMQP_BROKER_HOST": "testhost",
358+
"INVENIO_AMQP_BROKER_PORT": "5672",
359+
"INVENIO_AMQP_BROKER_PROTOCOL": "amqp",
360+
"INVENIO_AMQP_BROKER_VHOST": "",
361+
},
362+
"amqp://testuser:testpassword@testhost:5672/",
363+
),
364+
(
365+
{"INVENIO_BROKER_URL": "amqp://guest:guest@localhost:5672/"},
366+
"amqp://guest:guest@localhost:5672/",
367+
),
368+
(
369+
{},
370+
"amqp://guest:guest@localhost:5672/",
371+
),
372+
],
373+
)
374+
def test_build_broker_url_with_vhost(monkeypatch, env_vars, expected_url):
375+
"""Test building broker URL with vhost."""
376+
set_env_vars(monkeypatch, env_vars)
377+
assert build_broker_url() == expected_url
378+
379+
380+
@pytest.mark.parametrize(
381+
"env_vars, db, expected_url",
382+
[
383+
(
384+
{
385+
"INVENIO_KV_CACHE_HOST": "testhost",
386+
"INVENIO_KV_CACHE_PORT": "6379",
387+
"INVENIO_KV_CACHE_PASSWORD": "testpassword",
388+
"INVENIO_KV_CACHE_PROTOCOL": "redis",
389+
},
390+
2,
391+
"redis://:testpassword@testhost:6379/2",
392+
),
393+
(
394+
{
395+
"INVENIO_KV_CACHE_HOST": "testhost",
396+
"INVENIO_KV_CACHE_PORT": "6379",
397+
"INVENIO_KV_CACHE_PROTOCOL": "redis",
398+
},
399+
1,
400+
"redis://testhost:6379/1",
401+
),
402+
(
403+
{"BROKER_URL": "redis://localhost:6379/0"},
404+
None,
405+
"redis://localhost:6379/0",
406+
),
407+
(
408+
{"INVENIO_KV_CACHE_URL": "redis://localhost:6379/3"},
409+
3,
410+
"redis://localhost:6379/3",
411+
),
412+
(
413+
{},
414+
4,
415+
"redis://localhost:6379/4",
416+
),
417+
],
418+
)
419+
def test_build_redis_url(monkeypatch, env_vars, db, expected_url):
420+
"""Test building Redis URL."""
421+
set_env_vars(monkeypatch, env_vars)
422+
assert build_redis_url(db=db) == expected_url

0 commit comments

Comments
 (0)