prometheus: fix scrape duration growing unbounded (#13667)

plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java

@@ -23,6 +23,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import javax.inject.Inject;
@@ -101,6 +102,7 @@ public String toString() {
     }
 
     private static List<Item> metricsItems = new ArrayList<>();
+    private volatile long lastMetricsUpdateTime = 0L;
 
     @Inject
     private DataCenterDao dcDao;
@@ -488,7 +490,15 @@ private void addVMsBySizeMetrics(final List<Item> metricsList, final long dcId,
     }
 
     @Override
-    public void updateMetrics() {
+    public synchronized void updateMetrics() {
+        final long minIntervalMs = TimeUnit.SECONDS.toMillis(PrometheusExporterServer.PrometheusExporterMinRefreshInterval.value());
+        final long now = System.currentTimeMillis();
+        if (now - lastMetricsUpdateTime < minIntervalMs) {
+            logger.debug("Skipping metrics recomputation, last update was " + (now - lastMetricsUpdateTime) + "ms ago (min interval: " + minIntervalMs + "ms)");
+            return;
+        }
+
+        final long startNanos = System.nanoTime();
         final List<Item> latestMetricsItems = new ArrayList<Item>();
         try {
             for (final DataCenterVO dc : dcDao.listAll()) {
@@ -508,8 +518,12 @@ public void updateMetrics() {
             addDomainResourceCount(latestMetricsItems);
         } catch (Exception e) {
             logger.warn("Getting metrics failed ", e);
+        } finally {
+            final long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+            logger.info("Prometheus metrics update completed in " + elapsedMs + " ms");
         }
         metricsItems = latestMetricsItems;
+        lastMetricsUpdateTime = System.currentTimeMillis();
     }
 
     @Override

plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServer.java

@@ -33,4 +33,7 @@ public interface PrometheusExporterServer extends Manager {
 
     ConfigKey<Integer> PrometheusExporterOfferingCountLimit = new ConfigKey<>("Advanced", Integer.class, "prometheus.exporter.offering.output.limit", "-1",
             "Limit the number of output for cloudstack_vms_total_by_size to the provided value. -1 for unlimited output.", true);
+
+    ConfigKey<Integer> PrometheusExporterMinRefreshInterval = new ConfigKey<>("Advanced", Integer.class, "prometheus.exporter.metrics.min.refresh.interval", "5",
+            "Minimum interval in seconds between metrics recomputations. Scrapes arriving faster than this interval reuse the previously computed metrics.", true, EnablePrometheusExporter.key());
 }

plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java

@@ -29,10 +29,13 @@
 import java.net.InetSocketAddress;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 public class PrometheusExporterServerImpl extends ManagerBase implements PrometheusExporterServer, Configurable {
 
     private static HttpServer httpServer;
+    private ExecutorService httpExecutor;
 
     @Inject
     private PrometheusExporter prometheusExporter;
@@ -79,6 +82,8 @@ public boolean start() {
         if (EnablePrometheusExporter.value()) {
             try {
                 httpServer = HttpServer.create(new InetSocketAddress(PrometheusExporterServerPort.value()), 0);
+                httpExecutor = Executors.newFixedThreadPool(2);
+                httpServer.setExecutor(httpExecutor);
                 httpServer.createContext("/metrics", new ExporterHandler(prometheusExporter));
                 httpServer.createContext("/", new HttpHandler() {
                     @Override
@@ -105,9 +110,15 @@ public void handle(HttpExchange httpExchange) throws IOException {
     @Override
     public boolean stop() {
         if (httpServer != null) {
+            httpServer.setExecutor(null);
             httpServer.stop(0);
             logger.debug("Stopped Prometheus exporter http server");
         }
+        if (httpExecutor != null) {
+            httpExecutor.shutdownNow();
+            logger.debug("Shut down Prometheus exporter http executor");
+        }
+        httpExecutor = null;
         return true;
     }
 
@@ -122,7 +133,8 @@ public ConfigKey<?>[] getConfigKeys() {
                 EnablePrometheusExporter,
                 PrometheusExporterServerPort,
                 PrometheusExporterAllowedAddresses,
-                PrometheusExporterOfferingCountLimit
+                PrometheusExporterOfferingCountLimit,
+                PrometheusExporterMinRefreshInterval
         };
     }
 }

plugins/integrations/prometheus/src/test/java/org/apache/cloudstack/metrics/PrometheusExporterImplTest.java

@@ -18,6 +18,15 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.lang.reflect.Field;
+import java.util.Collections;
+
+import com.cloud.dc.dao.DataCenterDao;
 
 import org.junit.Test;
 
@@ -105,4 +114,66 @@ public void testItemHostCertExpiryContainsTimestampValue() {
         assertTrue("Metric should contain correct timestamp value",
                 metricsString.endsWith(" " + CERT_EXPIRY_EPOCH));
     }
+
+    /**
+     * Two rapid calls to updateMetrics() within the min refresh interval
+     * should result in only one actual recomputation (one call to dcDao.listAll()).
+     */
+    @Test
+    public void testUpdateMetricsTTLGuardSkipsSecondCall() throws Exception {
+        PrometheusExporterImpl exporter = new PrometheusExporterImpl();
+
+        DataCenterDao mockDcDao = mock(DataCenterDao.class);
+        when(mockDcDao.listAll()).thenReturn(Collections.emptyList());
+        setField(exporter, "dcDao", mockDcDao);
+
+        // First call should trigger recomputation
+        exporter.updateMetrics();
+        // Second immediate call should be skipped by the TTL guard
+        exporter.updateMetrics();
+
+        verify(mockDcDao, times(1)).listAll();
+    }
+
+    /**
+     * After the min refresh interval has elapsed, updateMetrics() should
+     * trigger a fresh recomputation.
+     */
+    @Test
+    public void testUpdateMetricsTTLGuardAllowsAfterInterval() throws Exception {
+        PrometheusExporterImpl exporter = new PrometheusExporterImpl();
+
+        DataCenterDao mockDcDao = mock(DataCenterDao.class);
+        when(mockDcDao.listAll()).thenReturn(Collections.emptyList());
+        setField(exporter, "dcDao", mockDcDao);
+
+        // First call
+        exporter.updateMetrics();
+
+        // Simulate that the min interval has already elapsed by resetting lastMetricsUpdateTime
+        setField(exporter, "lastMetricsUpdateTime", 0L);
+
+        // Second call should now trigger recomputation
+        exporter.updateMetrics();
+
+        verify(mockDcDao, times(2)).listAll();
+    }
+
+    private static void setField(Object target, String fieldName, Object value) throws Exception {
+        Field field = null;
+        Class<?> clazz = target.getClass();
+        while (clazz != null) {
+            try {
+                field = clazz.getDeclaredField(fieldName);
+                break;
+            } catch (NoSuchFieldException e) {
+                clazz = clazz.getSuperclass();
+            }
+        }
+        if (field == null) {
+            throw new NoSuchFieldException(fieldName);
+        }
+        field.setAccessible(true);
+        field.set(target, value);
+    }
 }