Skip to content

Commit ae49b00

Browse files
dricrossdricross
authored
Replace telegraf prometheus plugin (#1709)
1 parent 9415228 commit ae49b00

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+541
-1094
lines changed

‎plugins/plugins.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
//Enable cloudwatch-agent process plugins
88
_ "github.com/aws/amazon-cloudwatch-agent/plugins/processors/ecsdecorator"
99
_ "github.com/aws/amazon-cloudwatch-agent/plugins/processors/k8sdecorator"
10+
_ "github.com/aws/amazon-cloudwatch-agent/plugins/processors/prometheusadapter"
1011

1112
// Enabled cloudwatch-agent input plugins
1213
_ "github.com/aws/amazon-cloudwatch-agent/plugins/inputs/logfile"

‎plugins/processors/prometheusadapter/config.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package prometheusadapter
5+
6+
import (
7+
"go.opentelemetry.io/collector/component"
8+
)
9+
10+
type Config struct{}
11+
12+
// Verify Config implements Processor interface.
13+
var _ component.Config = (*Config)(nil)
14+
15+
func (cfg *Config) Validate() error {
16+
return nil
17+
}

‎plugins/processors/prometheusadapter/config_test.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package prometheusadapter
5+
6+
import (
7+
"testing"
8+
9+
"github.com/stretchr/testify/assert"
10+
"go.opentelemetry.io/collector/confmap"
11+
)
12+
13+
func TestUnmarshalDefaultConfig(t *testing.T) {
14+
factory := NewFactory()
15+
cfg := factory.CreateDefaultConfig()
16+
assert.NoError(t, confmap.New().Unmarshal(cfg))
17+
assert.Equal(t, factory.CreateDefaultConfig(), cfg)
18+
}

‎plugins/processors/prometheusadapter/factory.go

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package prometheusadapter
5+
6+
import (
7+
"context"
8+
"fmt"
9+
10+
"go.opentelemetry.io/collector/component"
11+
"go.opentelemetry.io/collector/consumer"
12+
"go.opentelemetry.io/collector/processor"
13+
"go.opentelemetry.io/collector/processor/processorhelper"
14+
)
15+
16+
const (
17+
stability = component.StabilityLevelBeta
18+
)
19+
20+
var (
21+
TypeStr, _ = component.NewType("prometheusadapter")
22+
processorCapabilities = consumer.Capabilities{MutatesData: true}
23+
)
24+
25+
func NewFactory() processor.Factory {
26+
return processor.NewFactory(
27+
TypeStr,
28+
createDefaultConfig,
29+
processor.WithMetrics(createMetricsProcessor, stability))
30+
}
31+
32+
func createDefaultConfig() component.Config {
33+
return &Config{}
34+
}
35+
36+
func createMetricsProcessor(
37+
ctx context.Context,
38+
set processor.Settings,
39+
cfg component.Config,
40+
nextConsumer consumer.Metrics,
41+
) (processor.Metrics, error) {
42+
processorConfig, ok := cfg.(*Config)
43+
if !ok {
44+
return nil, fmt.Errorf("configuration parsing error")
45+
}
46+
47+
metricsProcessor := newPrometheusAdapterProcessor(processorConfig, set.Logger)
48+
49+
return processorhelper.NewMetrics(
50+
ctx,
51+
set,
52+
cfg,
53+
nextConsumer,
54+
metricsProcessor.processMetrics,
55+
processorhelper.WithCapabilities(processorCapabilities))
56+
}

‎plugins/processors/prometheusadapter/factory_test.go

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package prometheusadapter
5+
6+
import (
7+
"context"
8+
"testing"
9+
10+
"github.com/stretchr/testify/assert"
11+
"github.com/stretchr/testify/require"
12+
"go.opentelemetry.io/collector/component/componenttest"
13+
"go.opentelemetry.io/collector/consumer/consumertest"
14+
"go.opentelemetry.io/collector/pipeline"
15+
"go.opentelemetry.io/collector/processor/processortest"
16+
)
17+
18+
func TestCreateDefaultConfig(t *testing.T) {
19+
factory := NewFactory()
20+
require.NotNil(t, factory)
21+
22+
cfg := factory.CreateDefaultConfig()
23+
assert.NotNil(t, cfg, "failed to create default config")
24+
assert.NoError(t, componenttest.CheckConfigStruct(cfg))
25+
}
26+
27+
func TestCreateProcessor(t *testing.T) {
28+
factory := NewFactory()
29+
require.NotNil(t, factory)
30+
31+
cfg := factory.CreateDefaultConfig()
32+
setting := processortest.NewNopSettings()
33+
34+
tProcessor, err := factory.CreateTraces(context.Background(), setting, cfg, consumertest.NewNop())
35+
assert.Equal(t, err, pipeline.ErrSignalNotSupported)
36+
assert.Nil(t, tProcessor)
37+
38+
mProcessor, err := factory.CreateMetrics(context.Background(), setting, cfg, consumertest.NewNop())
39+
assert.NoError(t, err)
40+
assert.NotNil(t, mProcessor)
41+
42+
lProcessor, err := factory.CreateLogs(context.Background(), setting, cfg, consumertest.NewNop())
43+
assert.Equal(t, err, pipeline.ErrSignalNotSupported)
44+
assert.Nil(t, lProcessor)
45+
}

‎plugins/processors/prometheusadapter/processor.go

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package prometheusadapter
5+
6+
import (
7+
"context"
8+
9+
"go.opentelemetry.io/collector/pdata/pmetric"
10+
"go.uber.org/zap"
11+
)
12+
13+
type prometheusAdapterProcessor struct {
14+
*Config
15+
logger *zap.Logger
16+
}
17+
18+
func newPrometheusAdapterProcessor(config *Config, logger *zap.Logger) *prometheusAdapterProcessor {
19+
d := &prometheusAdapterProcessor{
20+
Config: config,
21+
logger: logger,
22+
}
23+
return d
24+
}
25+
26+
func (d *prometheusAdapterProcessor) processMetrics(_ context.Context, md pmetric.Metrics) (pmetric.Metrics, error) {
27+
return md, nil
28+
}

‎plugins/processors/prometheusadapter/processor_test.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package prometheusadapter
5+
6+
import (
7+
"testing"
8+
9+
"github.com/stretchr/testify/assert"
10+
"go.uber.org/zap"
11+
)
12+
13+
func TestProcessMetricsForPrometheusAdapter(t *testing.T) {
14+
logger, _ := zap.NewDevelopment()
15+
pap := newPrometheusAdapterProcessor(createDefaultConfig().(*Config), logger)
16+
17+
assert.NotNil(t, pap)
18+
}

‎receiver/adapter/plugins_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ func scrapeMetrics(as *assert.Assertions, ctx context.Context, receiver *Adapted
124124
}
125125

126126
func validateMetricName(as *assert.Assertions, plugin string, expectedResourceMetricsName []string, actualOtelSlMetrics pmetric.MetricSlice) {
127-
as.Equal(len(expectedResourceMetricsName), actualOtelSlMetrics.Len(), "Number of metrics did not match!")
127+
as.Equal(len(expectedResourceMetricsName), actualOtelSlMetrics.Len(), "Number of metrics did not match! expected %+v\nactual: %+v", expectedResourceMetricsName, actualOtelSlMetrics)
128128

129129
matchMetrics := actualOtelSlMetrics.Len()
130130
for _, expectedMetric := range expectedResourceMetricsName {

‎service/defaultcomponents/components.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ import (
6565
"github.com/aws/amazon-cloudwatch-agent/plugins/processors/ec2tagger"
6666
"github.com/aws/amazon-cloudwatch-agent/plugins/processors/gpuattributes"
6767
"github.com/aws/amazon-cloudwatch-agent/plugins/processors/kueueattributes"
68+
"github.com/aws/amazon-cloudwatch-agent/plugins/processors/prometheusadapter"
6869
"github.com/aws/amazon-cloudwatch-agent/processor/rollupprocessor"
6970
"github.com/aws/amazon-cloudwatch-agent/receiver/awsebsnvmereceiver"
7071
)
@@ -112,6 +113,7 @@ func Factories() (otelcol.Factories, error) {
112113
metricsgenerationprocessor.NewFactory(),
113114
metricstransformprocessor.NewFactory(),
114115
probabilisticsamplerprocessor.NewFactory(),
116+
prometheusadapter.NewFactory(),
115117
resourceprocessor.NewFactory(),
116118
resourcedetectionprocessor.NewFactory(),
117119
rollupprocessor.NewFactory(),

‎service/defaultcomponents/components_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ func TestComponents(t *testing.T) {
5757
"k8sattributes",
5858
"memory_limiter",
5959
"metricstransform",
60+
"prometheusadapter",
6061
"resourcedetection",
6162
"resource",
6263
"rollup",

‎translator/registerrules/register_rules.go

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,6 @@ import (
1616
_ "github.com/aws/amazon-cloudwatch-agent/translator/translate/logs/logs_collected/windows_events/collect_list"
1717
_ "github.com/aws/amazon-cloudwatch-agent/translator/translate/logs/metrics_collected/ecs"
1818
_ "github.com/aws/amazon-cloudwatch-agent/translator/translate/logs/metrics_collected/kubernetes"
19-
_ "github.com/aws/amazon-cloudwatch-agent/translator/translate/logs/metrics_collected/prometheus"
20-
_ "github.com/aws/amazon-cloudwatch-agent/translator/translate/logs/metrics_collected/prometheus/ecsservicediscovery"
21-
_ "github.com/aws/amazon-cloudwatch-agent/translator/translate/logs/metrics_collected/prometheus/ecsservicediscovery/dockerlabel"
22-
_ "github.com/aws/amazon-cloudwatch-agent/translator/translate/logs/metrics_collected/prometheus/ecsservicediscovery/serviceendpoint"
23-
_ "github.com/aws/amazon-cloudwatch-agent/translator/translate/logs/metrics_collected/prometheus/ecsservicediscovery/taskdefinition"
2419
_ "github.com/aws/amazon-cloudwatch-agent/translator/translate/metrics/drop_origin"
2520
_ "github.com/aws/amazon-cloudwatch-agent/translator/translate/metrics/metric_decoration"
2621
_ "github.com/aws/amazon-cloudwatch-agent/translator/translate/metrics/metrics_collect/collectd"

‎translator/tocwconfig/sampleConfig/prometheus_combined_config_linux.conf

Lines changed: 0 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -14,44 +14,6 @@
1414
quiet = false
1515
round_interval = false
1616

17-
[inputs]
18-
19-
[[inputs.prometheus]]
20-
cluster_name = "TestCluster"
21-
prometheus_config_path = "{prometheusFileName}"
22-
[inputs.prometheus.ecs_service_discovery]
23-
sd_cluster_region = "us-west-2"
24-
sd_frequency = "1m"
25-
sd_result_file = "{ecsSdFileName}"
26-
sd_target_cluster = "ecs-cluster-a"
27-
[inputs.prometheus.ecs_service_discovery.docker_label]
28-
sd_job_name_label = "ECS_PROMETHEUS_JOB_NAME_1"
29-
sd_metrics_path_label = "ECS_PROMETHEUS_METRICS_PATH"
30-
sd_port_label = "ECS_PROMETHEUS_EXPORTER_PORT_SUBSET"
31-
32-
[[inputs.prometheus.ecs_service_discovery.service_name_list_for_tasks]]
33-
sd_container_name_pattern = "nginx-prometheus-exporter"
34-
sd_job_name = "service_name_1"
35-
sd_metrics_path = "/metrics"
36-
sd_metrics_ports = "9113"
37-
sd_service_name_pattern = ".*-application-stack"
38-
39-
[[inputs.prometheus.ecs_service_discovery.service_name_list_for_tasks]]
40-
sd_metrics_path = "/stats/metrics"
41-
sd_metrics_ports = "9114"
42-
sd_service_name_pattern = "run-application-stack"
43-
44-
[[inputs.prometheus.ecs_service_discovery.task_definition_list]]
45-
sd_job_name = "task_def_1"
46-
sd_metrics_path = "/stats/metrics"
47-
sd_metrics_ports = "9901"
48-
sd_task_definition_arn_pattern = ".*task_def_1:[0-9]+"
49-
50-
[[inputs.prometheus.ecs_service_discovery.task_definition_list]]
51-
sd_container_name_pattern = "^envoy$"
52-
sd_metrics_ports = "9902"
53-
sd_task_definition_arn_pattern = "task_def_2"
54-
5517
[outputs]
5618

5719
[[outputs.cloudwatch]]

‎translator/tocwconfig/sampleConfig/prometheus_combined_config_linux.yaml

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,7 @@ processors:
155155
deltatocumulative/prometheus/amp:
156156
max_stale: 336h0m0s
157157
max_streams: 9223372036854775807
158+
prometheusadapter/prometheus/cloudwatchlogs: {}
158159
receivers:
159160
prometheus:
160161
config:
@@ -190,10 +191,40 @@ receivers:
190191
start_time_metric_regex: ""
191192
trim_metric_suffixes: false
192193
use_start_time_metric: false
193-
telegraf_prometheus:
194-
collection_interval: 1m0s
195-
initial_delay: 1s
196-
timeout: 0s
194+
prometheus/prometheus/cloudwatchlogs:
195+
config:
196+
global:
197+
evaluation_interval: 1m
198+
scrape_interval: 5m
199+
scrape_protocols:
200+
- OpenMetricsText1.0.0
201+
- OpenMetricsText0.0.1
202+
- PrometheusText0.0.4
203+
scrape_timeout: 5s
204+
scrape_configs:
205+
- enable_compression: true
206+
enable_http2: true
207+
file_sd_configs:
208+
- files:
209+
- {ecsSdFileName}
210+
refresh_interval: 5m
211+
follow_redirects: true
212+
honor_timestamps: true
213+
job_name: cwagent-ecs-file-sd-config
214+
metrics_path: /metrics
215+
sample_limit: 10000
216+
scheme: http
217+
scrape_interval: 5m
218+
scrape_protocols:
219+
- OpenMetricsText1.0.0
220+
- OpenMetricsText0.0.1
221+
- PrometheusText0.0.4
222+
scrape_timeout: 5s
223+
track_timestamps_staleness: false
224+
report_extra_scrape_metrics: false
225+
start_time_metric_regex: ""
226+
trim_metric_suffixes: false
227+
use_start_time_metric: false
197228
service:
198229
extensions:
199230
- agenthealth/logs
@@ -213,9 +244,10 @@ service:
213244
exporters:
214245
- awsemf/prometheus
215246
processors:
247+
- prometheusadapter/prometheus/cloudwatchlogs
216248
- batch/prometheus/cloudwatchlogs
217249
receivers:
218-
- telegraf_prometheus
250+
- prometheus/prometheus/cloudwatchlogs
219251
telemetry:
220252
logs:
221253
development: false

‎translator/tocwconfig/sampleConfig/prometheus_config_linux.conf

Lines changed: 0 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -14,44 +14,6 @@
1414
quiet = false
1515
round_interval = false
1616

17-
[inputs]
18-
19-
[[inputs.prometheus]]
20-
cluster_name = "TestCluster"
21-
prometheus_config_path = "{prometheusFileName}"
22-
[inputs.prometheus.ecs_service_discovery]
23-
sd_cluster_region = "us-west-1"
24-
sd_frequency = "1m"
25-
sd_result_file = "{ecsSdFileName}"
26-
sd_target_cluster = "ecs-cluster-a"
27-
[inputs.prometheus.ecs_service_discovery.docker_label]
28-
sd_job_name_label = "ECS_PROMETHEUS_JOB_NAME_1"
29-
sd_metrics_path_label = "ECS_PROMETHEUS_METRICS_PATH"
30-
sd_port_label = "ECS_PROMETHEUS_EXPORTER_PORT_SUBSET"
31-
32-
[[inputs.prometheus.ecs_service_discovery.service_name_list_for_tasks]]
33-
sd_container_name_pattern = "nginx-prometheus-exporter"
34-
sd_job_name = "service_name_1"
35-
sd_metrics_path = "/metrics"
36-
sd_metrics_ports = "9113"
37-
sd_service_name_pattern = ".*-application-stack"
38-
39-
[[inputs.prometheus.ecs_service_discovery.service_name_list_for_tasks]]
40-
sd_metrics_path = "/stats/metrics"
41-
sd_metrics_ports = "9114"
42-
sd_service_name_pattern = "run-application-stack"
43-
44-
[[inputs.prometheus.ecs_service_discovery.task_definition_list]]
45-
sd_job_name = "task_def_1"
46-
sd_metrics_path = "/stats/metrics"
47-
sd_metrics_ports = "9901"
48-
sd_task_definition_arn_pattern = ".*task_def_1:[0-9]+"
49-
50-
[[inputs.prometheus.ecs_service_discovery.task_definition_list]]
51-
sd_container_name_pattern = "^envoy$"
52-
sd_metrics_ports = "9902"
53-
sd_task_definition_arn_pattern = "task_def_2"
54-
5517
[outputs]
5618

5719
[[outputs.cloudwatchlogs]]

0 commit comments

Comments
 (0)