Skip to content

Commit e52709c

Browse files
Samk13Samk13
committed
config: granular env-based solution for connection strings
Add logic to build connection urls from env vars if available needed for helm charts security best practices. includes: * Build db uri * Build redis url * Build mq url Partially closes: inveniosoftware/helm-invenio#112
1 parent e3bc51c commit e52709c

File tree

2 files changed

+287
-0
lines changed

2 files changed

+287
-0
lines changed

‎invenio_config/env.py

+98
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#
33
# This file is part of Invenio.
44
# Copyright (C) 2015-2018 CERN.
5+
# Copyright (C) 2024 KTH Royal Institute of Technology.
56
#
67
# Invenio is free software; you can redistribute it and/or modify it
78
# under the terms of the MIT License; see LICENSE file for more details.
@@ -45,3 +46,100 @@ def init_app(self, app):
4546

4647
# Set value
4748
app.config[varname] = value
49+
50+
51+
def _get_env_var(prefix, keys):
52+
"""Retrieve environment variables with a given prefix."""
53+
return {k: os.environ.get(f"{prefix}_{k.upper()}") for k in keys}
54+
55+
56+
def build_db_uri():
57+
"""
58+
Build database URI from environment variables or use default.
59+
60+
Priority order:
61+
1. INVENIO_SQLALCHEMY_DATABASE_URI
62+
2. SQLALCHEMY_DATABASE_URI
63+
3. INVENIO_DB_* specific environment variables
64+
4. Default URI
65+
66+
Note: For option 3, to assert that the INVENIO_DB_* settings take effect,
67+
you need to set SQLALCHEMY_DATABASE_URI="" in your environment.
68+
"""
69+
default_uri = "postgresql+psycopg2://invenio-app-rdm:invenio-app-rdm@localhost/invenio-app-rdm"
70+
71+
uri = os.environ.get("INVENIO_SQLALCHEMY_DATABASE_URI") or os.environ.get(
72+
"SQLALCHEMY_DATABASE_URI"
73+
)
74+
if uri:
75+
return uri
76+
77+
db_params = _get_env_var(
78+
"INVENIO_DB", ["user", "password", "host", "port", "name", "protocol"]
79+
)
80+
if all(db_params.values()):
81+
uri = f"{db_params['protocol']}://{db_params['user']}:{db_params['password']}@{db_params['host']}:{db_params['port']}/{db_params['name']}"
82+
return uri
83+
84+
return default_uri
85+
86+
87+
def build_broker_url():
88+
"""
89+
Build broker URL from environment variables or use default.
90+
91+
Priority order:
92+
1. INVENIO_BROKER_URL
93+
2. BROKER_URL
94+
3. INVENIO_BROKER_* specific environment variables
95+
4. Default URL
96+
"""
97+
default_url = "amqp://guest:guest@localhost:5672/"
98+
99+
uri = os.environ.get("INVENIO_BROKER_URL") or os.environ.get("BROKER_URL")
100+
if uri:
101+
return uri
102+
103+
broker_params = _get_env_var(
104+
"INVENIO_BROKER", ["user", "password", "host", "port", "protocol"]
105+
)
106+
if all(broker_params.values()):
107+
vhost = f"{os.environ.get("INVENIO_BROKER_VHOST").lstrip("/")}"
108+
return f"{broker_params['protocol']}://{broker_params['user']}:{broker_params['password']}@{broker_params['host']}:{broker_params['port']}/{vhost}"
109+
110+
return default_url
111+
112+
113+
def build_redis_url(db=None):
114+
"""
115+
Build Redis URL from environment variables or use default.
116+
117+
Priority order:
118+
1. BROKER_URL (Redis-based)
119+
2. INVENIO_REDIS_URL
120+
3. INVENIO_REDIS_* specific environment variables
121+
4. Default URL
122+
"""
123+
db = db if db is not None else 0
124+
default_url = f"redis://localhost:6379/{db}"
125+
126+
uri = os.environ.get("BROKER_URL")
127+
if uri and uri.startswith(("redis://", "rediss://", "unix://")):
128+
return uri
129+
130+
uri = os.environ.get("INVENIO_REDIS_URL")
131+
if uri:
132+
return uri
133+
134+
redis_params = _get_env_var(
135+
"INVENIO_REDIS", ["host", "port", "password", "protocol"]
136+
)
137+
redis_params["protocol"] = redis_params.get("protocol") or "redis"
138+
139+
if redis_params["host"] and redis_params["port"]:
140+
password = (
141+
f":{redis_params['password']}@" if redis_params.get("password") else ""
142+
)
143+
return f"{redis_params['protocol']}://{password}{redis_params['host']}:{redis_params['port']}/{db}"
144+
145+
return default_url

‎tests/test_invenio_config.py

+189
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#
33
# This file is part of Invenio.
44
# Copyright (C) 2015-2018 CERN.
5+
# Copyright (C) 2024 KTH Royal Institute of Technology.
56
#
67
# Invenio is free software; you can redistribute it and/or modify it
78
# under the terms of the MIT License; see LICENSE file for more details.
@@ -16,6 +17,7 @@
1617
import warnings
1718
from os.path import join
1819

20+
import pytest
1921
from flask import Flask
2022
from mock import patch
2123
from pkg_resources import EntryPoint
@@ -29,6 +31,7 @@
2931
create_config_loader,
3032
)
3133
from invenio_config.default import ALLOWED_HTML_ATTRS, ALLOWED_HTML_TAGS
34+
from invenio_config.env import build_broker_url, build_db_uri, build_redis_url
3235

3336

3437
class ConfigEP(EntryPoint):
@@ -231,3 +234,189 @@ class Config(object):
231234
assert app.config["ENV"] == "env"
232235
finally:
233236
shutil.rmtree(tmppath)
237+
238+
239+
def set_env_vars(monkeypatch, env_vars):
240+
"""Helper function to set environment variables."""
241+
for key in env_vars:
242+
monkeypatch.delenv(key, raising=False)
243+
for key, value in env_vars.items():
244+
monkeypatch.setenv(key, value)
245+
246+
247+
@pytest.mark.parametrize(
248+
"env_vars, expected_uri",
249+
[
250+
(
251+
{
252+
"INVENIO_DB_USER": "testuser",
253+
"INVENIO_DB_PASSWORD": "testpassword",
254+
"INVENIO_DB_HOST": "testhost",
255+
"INVENIO_DB_PORT": "5432",
256+
"INVENIO_DB_NAME": "testdb",
257+
"INVENIO_DB_PROTOCOL": "postgresql+psycopg2",
258+
},
259+
"postgresql+psycopg2://testuser:testpassword@testhost:5432/testdb",
260+
),
261+
(
262+
{
263+
"INVENIO_SQLALCHEMY_DATABASE_URI": "postgresql+psycopg2://testuser:testpassword@testhost:5432/testdb"
264+
},
265+
"postgresql+psycopg2://testuser:testpassword@testhost:5432/testdb",
266+
),
267+
(
268+
{
269+
"SQLALCHEMY_DATABASE_URI": "postgresql+psycopg2://testuser:testpassword@testhost:5432/testdb"
270+
},
271+
"postgresql+psycopg2://testuser:testpassword@testhost:5432/testdb",
272+
),
273+
(
274+
{},
275+
"postgresql+psycopg2://invenio-app-rdm:invenio-app-rdm@localhost/invenio-app-rdm",
276+
),
277+
],
278+
)
279+
def test_build_db_uri(monkeypatch, env_vars, expected_uri):
280+
"""Test building database URI."""
281+
set_env_vars(monkeypatch, env_vars)
282+
assert build_db_uri() == expected_uri
283+
284+
285+
@pytest.mark.parametrize(
286+
"env_vars, expected_url",
287+
[
288+
(
289+
{
290+
"INVENIO_BROKER_USER": "testuser",
291+
"INVENIO_BROKER_PASSWORD": "testpassword",
292+
"INVENIO_BROKER_HOST": "testhost",
293+
"INVENIO_BROKER_PORT": "5672",
294+
"INVENIO_BROKER_PROTOCOL": "amqp",
295+
"INVENIO_BROKER_VHOST": "/testvhost",
296+
},
297+
"amqp://testuser:testpassword@testhost:5672/testvhost",
298+
),
299+
(
300+
{
301+
"INVENIO_BROKER_USER": "testuser",
302+
"INVENIO_BROKER_PASSWORD": "testpassword",
303+
"INVENIO_BROKER_HOST": "testhost",
304+
"INVENIO_BROKER_PORT": "5672",
305+
"INVENIO_BROKER_PROTOCOL": "amqp",
306+
"INVENIO_BROKER_VHOST": "testvhost",
307+
},
308+
"amqp://testuser:testpassword@testhost:5672/testvhost",
309+
),
310+
(
311+
{
312+
"INVENIO_BROKER_USER": "testuser",
313+
"INVENIO_BROKER_PASSWORD": "testpassword",
314+
"INVENIO_BROKER_HOST": "testhost",
315+
"INVENIO_BROKER_PORT": "5672",
316+
"INVENIO_BROKER_PROTOCOL": "amqp",
317+
"INVENIO_BROKER_VHOST": "",
318+
},
319+
"amqp://testuser:testpassword@testhost:5672/",
320+
),
321+
(
322+
{
323+
"INVENIO_BROKER_USER": "testuser",
324+
"INVENIO_BROKER_PASSWORD": "testpassword",
325+
"INVENIO_BROKER_HOST": "testhost",
326+
"INVENIO_BROKER_PORT": "5672",
327+
"INVENIO_BROKER_PROTOCOL": "amqp",
328+
},
329+
"amqp://testuser:testpassword@testhost:5672/",
330+
),
331+
],
332+
)
333+
def test_build_broker_url_with_vhost(monkeypatch, env_vars, expected_url):
334+
"""Test building broker URL with vhost."""
335+
set_env_vars(monkeypatch, env_vars)
336+
assert build_broker_url() == expected_url
337+
338+
339+
@pytest.mark.parametrize(
340+
"env_vars, expected_url",
341+
[
342+
(
343+
{
344+
"INVENIO_BROKER_USER": "testuser",
345+
"INVENIO_BROKER_PASSWORD": "testpassword",
346+
"INVENIO_BROKER_HOST": "testhost",
347+
"INVENIO_BROKER_PORT": "5672",
348+
"INVENIO_BROKER_PROTOCOL": "amqp",
349+
"INVENIO_BROKER_VHOST": "/testvhost",
350+
},
351+
"amqp://testuser:testpassword@testhost:5672/testvhost",
352+
),
353+
(
354+
{
355+
"INVENIO_BROKER_USER": "testuser",
356+
"INVENIO_BROKER_PASSWORD": "testpassword",
357+
"INVENIO_BROKER_HOST": "testhost",
358+
"INVENIO_BROKER_PORT": "5672",
359+
"INVENIO_BROKER_PROTOCOL": "amqp",
360+
"INVENIO_BROKER_VHOST": "testvhost",
361+
},
362+
"amqp://testuser:testpassword@testhost:5672/testvhost",
363+
),
364+
(
365+
{"INVENIO_BROKER_URL": "amqp://guest:guest@localhost:5672/"},
366+
"amqp://guest:guest@localhost:5672/",
367+
),
368+
(
369+
{},
370+
"amqp://guest:guest@localhost:5672/",
371+
),
372+
],
373+
)
374+
def test_build_broker_url_with_vhost(monkeypatch, env_vars, expected_url):
375+
"""Test building broker URL with vhost."""
376+
set_env_vars(monkeypatch, env_vars)
377+
assert build_broker_url() == expected_url
378+
379+
380+
@pytest.mark.parametrize(
381+
"env_vars, db, expected_url",
382+
[
383+
(
384+
{
385+
"INVENIO_REDIS_HOST": "testhost",
386+
"INVENIO_REDIS_PORT": "6379",
387+
"INVENIO_REDIS_PASSWORD": "testpassword",
388+
"INVENIO_REDIS_PROTOCOL": "redis",
389+
},
390+
2,
391+
"redis://:testpassword@testhost:6379/2",
392+
),
393+
(
394+
{
395+
"INVENIO_REDIS_HOST": "testhost",
396+
"INVENIO_REDIS_PORT": "6379",
397+
"INVENIO_REDIS_PROTOCOL": "redis",
398+
},
399+
1,
400+
"redis://testhost:6379/1",
401+
),
402+
(
403+
{"BROKER_URL": "redis://localhost:6379/0"},
404+
None,
405+
"redis://localhost:6379/0",
406+
),
407+
(
408+
{"INVENIO_REDIS_URL": "redis://localhost:6379/3"},
409+
3,
410+
"redis://localhost:6379/3",
411+
),
412+
(
413+
{},
414+
4,
415+
"redis://localhost:6379/4",
416+
),
417+
],
418+
)
419+
def test_build_redis_url(monkeypatch, env_vars, db, expected_url):
420+
"""Test building Redis URL."""
421+
set_env_vars(monkeypatch, env_vars)
422+
assert build_redis_url(db=db) == expected_url

0 commit comments

Comments
 (0)