fix bug after bad refactoring, cleanup

Author: pront

src/config/loading/config_builder.rs

@@ -3,7 +3,9 @@ use std::collections::HashMap;
 use indexmap::IndexMap;
 use toml::value::Table;
 
-use super::{deserialize_table, interpolate_toml_table_with_secrets, loader};
+use super::{
+    deserialize_table, interpolate_toml_table_with_secrets, loader, resolve_environment_variables,
+};
 use super::{ComponentHint, Process};
 use crate::config::{
     ComponentKey, ConfigBuilder, EnrichmentTableOuter, SinkOuter, SourceOuter, TestDefinition,
@@ -34,6 +36,7 @@ impl ConfigBuilderLoader {
 impl Process for ConfigBuilderLoader {
     /// Prepares input for a `ConfigBuilder` by interpolating environment variables.
     fn postprocess(&mut self, table: Table) -> Result<Table, Vec<String>> {
+        let table = resolve_environment_variables(table)?;
         self.secrets
             .as_ref()
             .map(|secrets_map| interpolate_toml_table_with_secrets(&table, secrets_map))

src/config/loading/interpolation.rs

@@ -84,36 +84,35 @@ pub fn interpolate_toml_table_with_env_vars(
 }
 
 /// Returns a new TOML `Table` with all string values interpolated.
-/// Leaves original table untouched.
 pub fn interpolate_toml_table(
     table: &Table,
     vars: &HashMap<String, String>,
     interpolate_fn: InterpolateFn,
 ) -> Result<Table, Vec<String>> {
     let mut result = Table::new();
-    let mut all_errors = Vec::new();
+    let mut errors = Vec::new();
 
     for (key, value) in table {
         let new_key = match interpolate_fn(key, vars) {
             Ok(k) => k,
             Err(errs) => {
-                all_errors.extend(errs);
+                errors.extend(errs);
                 key.clone()
             }
         };
 
-        let new_value = match interpolate_toml_value(value, vars, &mut all_errors, interpolate_fn) {
+        let new_value = match interpolate_toml_value(value, vars, &mut errors, interpolate_fn) {
             Some(v) => v,
             None => value.clone(),
         };
 
         result.insert(new_key, new_value);
     }
 
-    if all_errors.is_empty() {
+    if errors.is_empty() {
         Ok(result)
     } else {
-        Err(all_errors)
+        Err(errors)
     }
 }
 
@@ -203,12 +202,12 @@ mod test {
     }
 
     #[test]
-    fn test_interpolate_toml_table_recursive() {
+    fn test_interpolate_toml_table() {
         let raw = indoc! {r#"
         # ${IN_COMMENT_BUT_DOES_NOT_EXIST}
         [[tests]]
             name = "${NAME}"
-            # noop = "SECRET[i_dont_exist]"
+            # This line should not cause a loading failure - "SECRET[backend_1.i_dont_exist]"
             [[tests.inputs]]
                 insert_at = "${UNDEFINED:-foo}"
                 type = "log"

src/config/loading/loader.rs

@@ -53,11 +53,14 @@ pub(super) mod process {
     /// in here with subtly different names that can be hidden from public view, hence why
     /// this is nested in a private mod.
     pub trait Process {
-        /// This is invoked after the input string deserialization. This can be a useful step to interpolate
+        /// This is invoked after input deserialization. This can be a useful step to interpolate
         /// environment variables or perform some other post-processing on the table.
         fn postprocess(&mut self, table: Table) -> Result<Table, Vec<String>>;
 
-        /// Calls into the `prepare` method, and deserializes a `Read` to a `T`.
+        /// Deserializes the input using the given format and runs postprocessing on the result.
+        ///
+        /// This reads the input into a string, deserializes it into a `Table`, and then
+        /// applies `postprocess` to the resulting table.
         fn load<R: Read>(&mut self, input: R, format: Format) -> Result<Table, Vec<String>> {
             let value = string_from_input(input)?;
             let table: Table = format::deserialize(&value, format)?;

src/config/loading/secret.rs

@@ -11,7 +11,7 @@ use toml::value::Table;
 use toml::Value;
 use vector_lib::config::ComponentKey;
 
-use crate::config::loading::interpolate_toml_table;
+use crate::config::loading::{interpolate_toml_table, resolve_environment_variables};
 use crate::{
     config::{
         loading::{deserialize_table, process::Process, ComponentHint, Loader},
@@ -90,6 +90,7 @@ impl SecretBackendLoader {
 
 impl Process for SecretBackendLoader {
     fn postprocess(&mut self, table: Table) -> Result<Table, Vec<String>> {
+        let table = resolve_environment_variables(table)?;
         collect_secret_keys_from_table(&table, &mut self.secret_keys);
         Ok(table)
     }