Ensure jobs have unique names to ensure all resources are scraped (#1060

)

config/promitor/scraper/metrics.yaml

@@ -59,6 +59,8 @@ metrics:
     resources:
     - namespace: promitor-messaging
       queueName: orders
+    - namespace: promitor-messaging
+      queueName: sales
     - namespace: promitor-messaging-other-subscription
       queueName: orders
       subscriptionId: 0329dd2a-59dc-4493-aa54-cb01cb027dc2

src/Promitor.Agents.Scraper/Extensions/IServiceCollectionExtensions.cs

@@ -1,4 +1,6 @@
-using System.Linq;
+using System;
+using System.Linq;
+using System.Text;
 using System.Threading.Tasks;
 using JustEat.StatsD;
 using Microsoft.Extensions.Configuration;
@@ -22,6 +24,7 @@
 using Promitor.Agents.Scraper.Validation;
 using Promitor.Core.Metrics;
 using Promitor.Core.Metrics.Sinks;
+using Promitor.Core.Scraping.Configuration.Model.Metrics;
 using Promitor.Core.Scraping.Configuration.Runtime;
 using Promitor.Core.Scraping.Interfaces;
 using Promitor.Integrations.AzureMonitor.Configuration;
@@ -60,8 +63,7 @@ public static IServiceCollection ScheduleMetricScraping(this IServiceCollection
                     var resourceSubscriptionId = string.IsNullOrWhiteSpace(resource.SubscriptionId) ? metrics.AzureMetadata.SubscriptionId : resource.SubscriptionId;
                     var azureMonitorClient = azureMonitorClientFactory.CreateIfNotExists(metrics.AzureMetadata.Cloud, metrics.AzureMetadata.TenantId, resourceSubscriptionId, metricSinkWriter, runtimeMetricCollector, configuration, azureMonitorLoggingConfiguration, loggerFactory);
                     var scrapeDefinition = metric.CreateScrapeDefinition(resource, metrics.AzureMetadata);
-
-                    var jobName = $"{scrapeDefinition.SubscriptionId}-{scrapeDefinition.PrometheusMetricDefinition.Name}";
+                    var jobName = GenerateJobName(scrapeDefinition, resource);
 
                     services.AddScheduler(builder =>
                     {
@@ -87,6 +89,22 @@ public static IServiceCollection ScheduleMetricScraping(this IServiceCollection
             return services;
         }
 
+        private static string GenerateJobName(ScrapeDefinition<IAzureResourceDefinition> scrapeDefinition, IAzureResourceDefinition resource)
+        {
+            var jobNameBuilder = new StringBuilder();
+            jobNameBuilder.Append(scrapeDefinition.SubscriptionId);
+            jobNameBuilder.Append("-");
+            jobNameBuilder.Append(scrapeDefinition.ResourceGroupName);
+            jobNameBuilder.Append("-");
+            jobNameBuilder.Append(scrapeDefinition.PrometheusMetricDefinition.Name);
+            jobNameBuilder.Append("-");
+            jobNameBuilder.Append(resource.GetUniqueName());
+            jobNameBuilder.Append("-");
+            jobNameBuilder.Append(Guid.NewGuid().ToString());
+
+            return jobNameBuilder.ToString();
+        }
+
         /// <summary>
         ///     Defines the dependencies that Promitor requires
         /// </summary>

src/Promitor.Core.Scraping/Configuration/Model/Metrics/AzureResourceDefinition.cs

@@ -38,5 +38,8 @@ protected AzureResourceDefinition(ResourceType resourceType, string subscription
 
         /// <inheritdoc />
         public abstract string GetResourceName();
+
+        /// <inheritdoc />
+        public abstract string GetUniqueName();
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/IAzureResourceDefinition.cs

@@ -31,5 +31,10 @@ public interface IAzureResourceDefinition
         ///     For an Azure SQL Database it should be the name of the DB, not the server
         /// </example>
         string GetResourceName();
+
+        /// <summary>
+        ///     Gets a unique name for the resource
+        /// </summary>
+        string GetUniqueName();
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/ApiManagementResourceDefinition.cs

@@ -29,5 +29,8 @@ public ApiManagementResourceDefinition(string subscriptionId, string resourceGro
 
         /// <inheritdoc />
         public override string GetResourceName() => InstanceName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => $"{InstanceName}-{LocationName}";
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/AppPlanResourceDefinition.cs

@@ -15,5 +15,8 @@ public AppPlanResourceDefinition(string subscriptionId, string resourceGroupName
 
         /// <inheritdoc />
         public override string GetResourceName() => AppPlanName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => GetResourceName();
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/BlobStorageResourceDefinition.cs

@@ -24,5 +24,8 @@ public BlobStorageResourceDefinition(string subscriptionId, string resourceGroup
 
         /// <inheritdoc />
         public override string GetResourceName() => AccountName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => GetResourceName();
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/ContainerInstanceResourceDefinition.cs

@@ -12,5 +12,8 @@ public ContainerInstanceResourceDefinition(string subscriptionId, string resourc
 
         /// <inheritdoc />
         public override string GetResourceName() => ContainerGroup;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => GetResourceName();
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/ContainerRegistryResourceDefinition.cs

@@ -12,5 +12,8 @@ public ContainerRegistryResourceDefinition(string subscriptionId, string resourc
 
         /// <inheritdoc />
         public override string GetResourceName() => RegistryName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => GetResourceName();
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/CosmosDbResourceDefinition.cs

@@ -12,5 +12,8 @@ public CosmosDbResourceDefinition(string subscriptionId, string resourceGroupNam
 
         /// <inheritdoc />
         public override string GetResourceName() => DbName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => GetResourceName();
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/DeviceProvisioningServiceResourceDefinition.cs

@@ -12,5 +12,8 @@ public DeviceProvisioningServiceResourceDefinition(string subscriptionId, string
 
         /// <inheritdoc />
         public override string GetResourceName() => DeviceProvisioningServiceName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => GetResourceName();
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/FileStorageResourceDefinition.cs

@@ -24,5 +24,8 @@ public FileStorageResourceDefinition(string subscriptionId, string resourceGroup
 
         /// <inheritdoc />
         public override string GetResourceName() => AccountName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => GetResourceName();
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/FunctionAppResourceDefinition.cs

@@ -20,5 +20,8 @@ public FunctionAppResourceDefinition(string subscriptionId, string resourceGroup
 
         /// <inheritdoc />
         public override string GetResourceName() => FunctionAppName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => $"{FunctionAppName}-{SlotName}";
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/GenericAzureResourceDefinition.cs

@@ -13,6 +13,9 @@ public GenericAzureResourceDefinition(string subscriptionId, string resourceGrou
         public string ResourceUri { get; }
 
         /// <inheritdoc />
-        public override string GetResourceName() => null;
+        public override string GetResourceName() => ResourceUri;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => $"{ResourceUri}";
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/IoTHubResourceDefinition.cs

@@ -12,5 +12,8 @@ public IoTHubResourceDefinition(string subscriptionId, string resourceGroupName,
 
         /// <inheritdoc />
         public override string GetResourceName() => IoTHubName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => GetResourceName();
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/KeyVaultResourceDefinition.cs

@@ -12,5 +12,8 @@ public KeyVaultResourceDefinition(string subscriptionId, string resourceGroupNam
 
         /// <inheritdoc />
         public override string GetResourceName() => VaultName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => GetResourceName();
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/NetworkInterfaceResourceDefinition.cs

@@ -12,5 +12,8 @@ public NetworkInterfaceResourceDefinition(string subscriptionId, string resource
 
         /// <inheritdoc />
         public override string GetResourceName() => NetworkInterfaceName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => GetResourceName();
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/PostgreSqlResourceDefinition.cs

@@ -12,5 +12,8 @@ public PostgreSqlResourceDefinition(string subscriptionId, string resourceGroupN
 
         /// <inheritdoc />
         public override string GetResourceName() => ServerName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => GetResourceName();
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/RedisCacheResourceDefinition.cs

@@ -12,5 +12,8 @@ public RedisCacheResourceDefinition(string subscriptionId, string resourceGroupN
 
         /// <inheritdoc />
         public override string GetResourceName() => CacheName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => GetResourceName();
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/ServiceBusQueueResourceDefinition.cs

@@ -13,6 +13,9 @@ public ServiceBusQueueResourceDefinition(string subscriptionId, string resourceG
         public string QueueName { get; }
 
         /// <inheritdoc />
-        public override string GetResourceName() => QueueName;
+        public override string GetResourceName() => Namespace;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => $"{Namespace}-{QueueName}";
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/SqlDatabaseResourceDefinition.cs

@@ -31,5 +31,8 @@ public SqlDatabaseResourceDefinition(string subscriptionId, string resourceGroup
 
         /// <inheritdoc />
         public override string GetResourceName() => DatabaseName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => $"{ServerName}-{DatabaseName}";
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/SqlManagedInstanceResourceDefinition.cs

@@ -24,5 +24,8 @@ public SqlManagedInstanceResourceDefinition(string subscriptionId, string resour
 
         /// <inheritdoc />
         public override string GetResourceName() => InstanceName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => GetResourceName();
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/SqlServerResourceDefinition.cs

@@ -24,5 +24,8 @@ public SqlServerResourceDefinition(string subscriptionId, string resourceGroupNa
 
         /// <inheritdoc />
         public override string GetResourceName() => ServerName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => GetResourceName();
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/StorageAccountResourceDefinition.cs

@@ -24,5 +24,8 @@ public StorageAccountResourceDefinition(string subscriptionId, string resourceGr
 
         /// <inheritdoc />
         public override string GetResourceName() => AccountName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => GetResourceName();
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/StorageQueueResourceDefinition.cs

@@ -16,5 +16,8 @@ public StorageQueueResourceDefinition(string subscriptionId, string resourceGrou
 
         /// <inheritdoc />
         public override string GetResourceName() => AccountName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => $"{AccountName}-{QueueName}";
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/VirtualMachineResourceDefinition.cs

@@ -12,5 +12,8 @@ public VirtualMachineResourceDefinition(string subscriptionId, string resourceGr
 
         /// <inheritdoc />
         public override string GetResourceName() => VirtualMachineName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => GetResourceName();
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/VirtualMachineScaleSetsResourceDefinition.cs

@@ -12,5 +12,8 @@ public VirtualMachineScaleSetResourceDefinition(string subscriptionId, string re
 
         /// <inheritdoc />
         public override string GetResourceName() => ScaleSetName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => GetResourceName();
     }
 }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/WebAppResourceDefinition.cs

@@ -21,5 +21,8 @@ public WebAppResourceDefinition(string subscriptionId, string resourceGroupName,
 
         /// <inheritdoc />
         public override string GetResourceName() => WebAppName;
+
+        /// <inheritdoc />
+        public override string GetUniqueName() => $"{WebAppName}-{SlotName}";
     }
 }

src/Promitor.Integrations.Sinks.Prometheus/PrometheusScrapingEndpointMetricSink.cs

@@ -42,7 +42,7 @@ public async Task ReportMetricAsync(string metricName, string metricDescription,
 
             foreach (var measuredMetric in scrapeResult.MetricValues)
             {
-                var metricValue = measuredMetric.Value ?? 0;
+                var metricValue = DetermineMetricMeasurement(measuredMetric);
                 var metricDefinition = _metricsDeclarationProvider.GetPrometheusDefinition(metricName);
 
                 var metricLabels = DetermineLabels(metricDefinition, scrapeResult, measuredMetric);
@@ -54,6 +54,12 @@ public async Task ReportMetricAsync(string metricName, string metricDescription,
             await Task.WhenAll(reportMetricTasks);
         }
 
+        private double DetermineMetricMeasurement(MeasuredMetric scrapedMetricResult)
+        {
+            var metricUnavailableValue = _prometheusConfiguration.CurrentValue?.MetricUnavailableValue ?? Defaults.Prometheus.MetricUnavailableValue;
+            return scrapedMetricResult.Value ?? metricUnavailableValue;
+        }
+
         public Task ReportMetricAsync(string metricName, string metricDescription, double metricValue, Dictionary<string, string> labels)
         {
             Guard.NotNullOrEmpty(metricName, nameof(metricName));