From 3ea7736a8ced8f35e7faa51c6378cdcfcd810935 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Thu, 19 Feb 2026 21:13:42 +0530 Subject: [PATCH 01/14] feat(health,version): add health and version endpoints --- pom.xml | 26 ++ .../controller/health/HealthController.java | 85 +++++ .../controller/version/VersionController.java | 68 ++-- .../service/health/HealthService.java | 295 ++++++++++++++++++ .../utils/JwtUserIdValidationFilter.java | 4 +- 5 files changed, 444 insertions(+), 34 deletions(-) create mode 100644 src/main/java/com/iemr/inventory/controller/health/HealthController.java create mode 100644 src/main/java/com/iemr/inventory/service/health/HealthService.java diff --git a/pom.xml b/pom.xml index 27582b5f..1dc40d8a 100644 --- a/pom.xml +++ b/pom.xml @@ -403,6 +403,32 @@ + + io.github.git-commit-id + git-commit-id-maven-plugin + 9.0.2 + + + get-the-git-infos + + revision + + initialize + + + + true + ${project.build.outputDirectory}/git.properties + + ^git.branch$ + ^git.commit.id.abbrev$ + ^git.build.version$ + ^git.build.time$ + + false + false + + org.springframework.boot spring-boot-maven-plugin diff --git a/src/main/java/com/iemr/inventory/controller/health/HealthController.java b/src/main/java/com/iemr/inventory/controller/health/HealthController.java new file mode 100644 index 00000000..e2d9438b --- /dev/null +++ b/src/main/java/com/iemr/inventory/controller/health/HealthController.java @@ -0,0 +1,85 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +package com.iemr.inventory.controller.health; + +import java.time.Instant; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import com.iemr.inventory.service.health.HealthService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; + +@RestController +@RequestMapping("/health") +@Tag(name = "Health Check", description = "APIs for checking infrastructure health status") +public class HealthController { + + private static final Logger logger = LoggerFactory.getLogger(HealthController.class); + + private final HealthService healthService; + + public HealthController(HealthService healthService) { + this.healthService = healthService; + } + + @GetMapping + @Operation(summary = "Check infrastructure health", + description = "Returns the health status of MySQL, Redis, and other configured services") + @ApiResponses({ + @ApiResponse(responseCode = "200", description = "All checked components are UP"), + @ApiResponse(responseCode = "503", description = "One or more critical services are DOWN") + }) + public ResponseEntity> checkHealth() { + logger.info("Health check endpoint called"); + + try { + Map healthStatus = healthService.checkHealth(); + String overallStatus = (String) healthStatus.get("status"); + + HttpStatus httpStatus = "UP".equals(overallStatus) ? HttpStatus.OK : HttpStatus.SERVICE_UNAVAILABLE; + + logger.debug("Health check completed with status: {}", overallStatus); + return new ResponseEntity<>(healthStatus, httpStatus); + + } catch (Exception e) { + logger.error("Unexpected error during health check", e); + + Map errorResponse = Map.of( + "status", "DOWN", + "timestamp", Instant.now().toString() + ); + + return new ResponseEntity<>(errorResponse, HttpStatus.SERVICE_UNAVAILABLE); + } + } +} + + diff --git a/src/main/java/com/iemr/inventory/controller/version/VersionController.java b/src/main/java/com/iemr/inventory/controller/version/VersionController.java index 472a0bb9..3fdab115 100644 --- a/src/main/java/com/iemr/inventory/controller/version/VersionController.java +++ b/src/main/java/com/iemr/inventory/controller/version/VersionController.java @@ -1,8 +1,8 @@ /* -* AMRIT – Accessible Medical Records via Integrated Technology -* Integrated EHR (Electronic Health Records) Solution +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution * -* Copyright (C) "Piramal Swasthya Management and Research Institute" +* Copyright (C) "Piramal Swasthya Management and Research Institute" * * This file is part of AMRIT. * @@ -21,57 +21,59 @@ */ package com.iemr.inventory.controller.version; -import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; -import com.iemr.inventory.utils.response.OutputResponse; - -import io.swagger.annotations.ApiOperation; +import io.swagger.v3.oas.annotations.Operation; @RestController public class VersionController { - private Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + private final Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + + private static final String UNKNOWN_VALUE = "unknown"; - @ApiOperation(value = "Get version details", consumes = "application/json", produces = "application/json") - @RequestMapping(value = "/version", method = { RequestMethod.GET }) - public String versionInformation() { - OutputResponse output = new OutputResponse(); + @Operation(summary = "Get version information") + @GetMapping(value = "/version", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity> versionInformation() { + Map response = new LinkedHashMap<>(); try { logger.info("version Controller Start"); - output.setResponse(readGitProperties()); + Properties gitProperties = loadGitProperties(); + response.put("buildTimestamp", gitProperties.getProperty("git.build.time", UNKNOWN_VALUE)); + response.put("version", gitProperties.getProperty("git.build.version", UNKNOWN_VALUE)); + response.put("branch", gitProperties.getProperty("git.branch", UNKNOWN_VALUE)); + response.put("commitHash", gitProperties.getProperty("git.commit.id.abbrev", UNKNOWN_VALUE)); } catch (Exception e) { - output.setError(e); + logger.error("Failed to load version information", e); + response.put("buildTimestamp", UNKNOWN_VALUE); + response.put("version", UNKNOWN_VALUE); + response.put("branch", UNKNOWN_VALUE); + response.put("commitHash", UNKNOWN_VALUE); } - logger.info("version Controller End"); - return output.toString(); - } - - private String readGitProperties() throws Exception { - ClassLoader classLoader = getClass().getClassLoader(); - InputStream inputStream = classLoader.getResourceAsStream("git.properties"); - - return readFromInputStream(inputStream); + return ResponseEntity.ok(response); } - private String readFromInputStream(InputStream inputStream) throws IOException { - StringBuilder resultStringBuilder = new StringBuilder(); - try (BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) { - String line; - while ((line = br.readLine()) != null) { - resultStringBuilder.append(line).append("\n"); + private Properties loadGitProperties() throws IOException { + Properties properties = new Properties(); + try (InputStream input = getClass().getClassLoader() + .getResourceAsStream("git.properties")) { + if (input != null) { + properties.load(input); } } - return resultStringBuilder.toString(); + return properties; } } diff --git a/src/main/java/com/iemr/inventory/service/health/HealthService.java b/src/main/java/com/iemr/inventory/service/health/HealthService.java new file mode 100644 index 00000000..979e026d --- /dev/null +++ b/src/main/java/com/iemr/inventory/service/health/HealthService.java @@ -0,0 +1,295 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +package com.iemr.inventory.service.health; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Supplier; +import jakarta.annotation.PreDestroy; +import javax.sql.DataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.stereotype.Service; + +@Service +public class HealthService { + + private static final Logger logger = LoggerFactory.getLogger(HealthService.class); + + // Status constants + private static final String STATUS_KEY = "status"; + private static final String STATUS_UP = "UP"; + private static final String STATUS_DOWN = "DOWN"; + private static final String STATUS_DEGRADED = "DEGRADED"; + + // Severity levels + private static final String SEVERITY_OK = "OK"; + private static final String SEVERITY_WARNING = "WARNING"; + private static final String SEVERITY_CRITICAL = "CRITICAL"; + + // Timeouts (in seconds) + private static final long MYSQL_TIMEOUT_SECONDS = 3; + private static final long REDIS_TIMEOUT_SECONDS = 3; + + // Performance threshold (milliseconds) - response time > 2000ms = DEGRADED + private static final long RESPONSE_TIME_THRESHOLD_MS = 2000; + + private final DataSource dataSource; + private final RedisTemplate redisTemplate; + private final ExecutorService executorService; + + public HealthService(DataSource dataSource, + @Autowired(required = false) RedisTemplate redisTemplate) { + this.dataSource = dataSource; + this.redisTemplate = redisTemplate; + this.executorService = Executors.newFixedThreadPool(2); + } + + @PreDestroy + public void shutdown() { + if (executorService != null && !executorService.isShutdown()) { + try { + executorService.shutdown(); + if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) { + executorService.shutdownNow(); + logger.warn("ExecutorService did not terminate gracefully"); + } + } catch (InterruptedException e) { + executorService.shutdownNow(); + Thread.currentThread().interrupt(); + logger.warn("ExecutorService shutdown interrupted", e); + } + } + } + + public Map checkHealth() { + Map response = new LinkedHashMap<>(); + response.put("timestamp", Instant.now().toString()); + + Map> components = new LinkedHashMap<>(); + + // Check MySQL + Map mysqlStatus = new LinkedHashMap<>(); + performHealthCheck("MySQL", mysqlStatus, this::checkMySQLHealth); + components.put("mysql", mysqlStatus); + + // Check Redis + Map redisStatus = new LinkedHashMap<>(); + performHealthCheck("Redis", redisStatus, this::checkRedisHealth); + components.put("redis", redisStatus); + + response.put("components", components); + + // Compute overall status + String overallStatus = computeOverallStatus(components); + response.put(STATUS_KEY, overallStatus); + + return response; + } + + private HealthCheckResult checkMySQLHealth() { + CompletableFuture future = CompletableFuture.supplyAsync(() -> { + try (Connection connection = dataSource.getConnection(); + PreparedStatement stmt = connection.prepareStatement("SELECT 1 as health_check")) { + + stmt.setQueryTimeout((int) MYSQL_TIMEOUT_SECONDS); + + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + return new HealthCheckResult(true, null); + } + } + + return new HealthCheckResult(false, "No result from health check query"); + + } catch (Exception e) { + logger.warn("MySQL health check failed: {}", e.getMessage(), e); + return new HealthCheckResult(false, "MySQL connection failed"); + } + }, executorService); + + try { + return future.get(MYSQL_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } catch (TimeoutException e) { + future.cancel(true); + logger.warn("MySQL health check timed out after {} seconds", MYSQL_TIMEOUT_SECONDS); + return new HealthCheckResult(false, "MySQL health check timed out"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("MySQL health check was interrupted"); + return new HealthCheckResult(false, "MySQL health check was interrupted"); + } catch (Exception e) { + logger.warn("MySQL health check failed: {}", e.getMessage(), e); + return new HealthCheckResult(false, "MySQL connection failed"); + } + } + + private HealthCheckResult checkRedisHealth() { + if (redisTemplate == null) { + return new HealthCheckResult(true, "Redis not configured — skipped"); + } + + CompletableFuture future = null; + try { + future = CompletableFuture.supplyAsync(() -> { + try { + return redisTemplate.execute((org.springframework.data.redis.core.RedisCallback) (connection) -> connection.ping()); + } catch (Exception e) { + logger.debug("Redis PING failed: {}", e.getMessage(), e); + return null; + } + }, executorService); + + String pong = future.get(REDIS_TIMEOUT_SECONDS, TimeUnit.SECONDS); + + if ("PONG".equals(pong)) { + return new HealthCheckResult(true, null); + } + + return new HealthCheckResult(false, "Redis PING failed"); + + } catch (TimeoutException e) { + if (future != null) { + future.cancel(true); + } + logger.warn("Redis health check timed out after {} seconds", REDIS_TIMEOUT_SECONDS); + return new HealthCheckResult(false, "Redis health check timed out"); + } catch (InterruptedException e) { + logger.warn("Redis health check was interrupted"); + Thread.currentThread().interrupt(); + return new HealthCheckResult(false, "Redis health check was interrupted"); + } catch (Exception e) { + logger.warn("Redis health check failed: {}", e.getMessage(), e); + return new HealthCheckResult(false, "Redis connection failed"); + } + } + + private Map performHealthCheck(String componentName, + Map status, + Supplier checker) { + long startTime = System.currentTimeMillis(); + + try { + HealthCheckResult result = checker.get(); + long responseTime = System.currentTimeMillis() - startTime; + + // Set status + status.put(STATUS_KEY, result.isHealthy ? STATUS_UP : STATUS_DOWN); + + // Set response time + status.put("responseTimeMs", responseTime); + + // Determine severity based on response time and status + String severity = determineSeverity(result.isHealthy, responseTime); + status.put("severity", severity); + + // Include error message if present (sanitized) + if (result.error != null) { + status.put("error", result.error); + } + + return status; + + } catch (Exception e) { + long responseTime = System.currentTimeMillis() - startTime; + logger.error("{} health check failed with exception: {}", componentName, e.getMessage(), e); + + status.put(STATUS_KEY, STATUS_DOWN); + status.put("responseTimeMs", responseTime); + status.put("severity", SEVERITY_CRITICAL); + status.put("error", "Health check failed with an unexpected error"); + + return status; + } + } + + /** + * Determine severity level based on component health and response time. + * + * @param isHealthy true if component is healthy + * @param responseTimeMs response time in milliseconds + * @return severity level: OK, WARNING, or CRITICAL + */ + private String determineSeverity(boolean isHealthy, long responseTimeMs) { + if (!isHealthy) { + return SEVERITY_CRITICAL; + } + + if (responseTimeMs > RESPONSE_TIME_THRESHOLD_MS) { + return SEVERITY_WARNING; + } + + return SEVERITY_OK; + } + + private String computeOverallStatus(Map> components) { + boolean hasCritical = false; + boolean hasWarning = false; + + for (Map componentStatus : components.values()) { + String status = (String) componentStatus.get(STATUS_KEY); + String severity = (String) componentStatus.get("severity"); + + if (STATUS_DOWN.equals(status) || SEVERITY_CRITICAL.equals(severity)) { + hasCritical = true; + } + + if (SEVERITY_WARNING.equals(severity)) { + hasWarning = true; + } + } + + if (hasCritical) { + return STATUS_DOWN; + } + + if (hasWarning) { + return STATUS_DEGRADED; + } + + return STATUS_UP; + } + + private static class HealthCheckResult { + final boolean isHealthy; + final String error; + + HealthCheckResult(boolean isHealthy, String error) { + this.isHealthy = isHealthy; + this.error = error; + } + } +} + + diff --git a/src/main/java/com/iemr/inventory/utils/JwtUserIdValidationFilter.java b/src/main/java/com/iemr/inventory/utils/JwtUserIdValidationFilter.java index 6f37aa8b..80b857c8 100644 --- a/src/main/java/com/iemr/inventory/utils/JwtUserIdValidationFilter.java +++ b/src/main/java/com/iemr/inventory/utils/JwtUserIdValidationFilter.java @@ -113,7 +113,9 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo || path.startsWith(contextPath + "/swagger-ui") || path.startsWith(contextPath + "/v3/api-docs") || path.startsWith(contextPath + "/user/refreshToken") - || path.startsWith(contextPath + "/public")) { + || path.startsWith(contextPath + "/public") + || path.equals(contextPath + "/version") + || path.equals(contextPath + "/health")) { logger.info("Skipping filter for path: " + path); filterChain.doFilter(servletRequest, servletResponse); return; From a07d45a41ae2d1b86cb8daa03e26f60cab4f85ca Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Thu, 19 Feb 2026 21:18:34 +0530 Subject: [PATCH 02/14] fix(health): removed duplicates from healthservices --- .../com/iemr/inventory/service/health/HealthService.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/iemr/inventory/service/health/HealthService.java b/src/main/java/com/iemr/inventory/service/health/HealthService.java index 979e026d..2d4ec30e 100644 --- a/src/main/java/com/iemr/inventory/service/health/HealthService.java +++ b/src/main/java/com/iemr/inventory/service/health/HealthService.java @@ -53,7 +53,8 @@ public class HealthService { private static final String STATUS_DOWN = "DOWN"; private static final String STATUS_DEGRADED = "DEGRADED"; - // Severity levels + // Severity levels and keys + private static final String SEVERITY_KEY = "severity"; private static final String SEVERITY_OK = "OK"; private static final String SEVERITY_WARNING = "WARNING"; private static final String SEVERITY_CRITICAL = "CRITICAL"; @@ -212,7 +213,7 @@ private Map performHealthCheck(String componentName, // Determine severity based on response time and status String severity = determineSeverity(result.isHealthy, responseTime); - status.put("severity", severity); + status.put(SEVERITY_KEY, severity); // Include error message if present (sanitized) if (result.error != null) { @@ -227,7 +228,7 @@ private Map performHealthCheck(String componentName, status.put(STATUS_KEY, STATUS_DOWN); status.put("responseTimeMs", responseTime); - status.put("severity", SEVERITY_CRITICAL); + status.put(SEVERITY_KEY, SEVERITY_CRITICAL); status.put("error", "Health check failed with an unexpected error"); return status; @@ -259,7 +260,7 @@ private String computeOverallStatus(Map> components) for (Map componentStatus : components.values()) { String status = (String) componentStatus.get(STATUS_KEY); - String severity = (String) componentStatus.get("severity"); + String severity = (String) componentStatus.get(SEVERITY_KEY); if (STATUS_DOWN.equals(status) || SEVERITY_CRITICAL.equals(severity)) { hasCritical = true; From 08f3b8ba9e3c5a2d7c9eb452cbc1e78a72c125b6 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Thu, 19 Feb 2026 21:27:51 +0530 Subject: [PATCH 03/14] fix: The DEGRADED status was incorrectly returning HTTP 503 --- .../com/iemr/inventory/controller/health/HealthController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iemr/inventory/controller/health/HealthController.java b/src/main/java/com/iemr/inventory/controller/health/HealthController.java index e2d9438b..a085d04a 100644 --- a/src/main/java/com/iemr/inventory/controller/health/HealthController.java +++ b/src/main/java/com/iemr/inventory/controller/health/HealthController.java @@ -64,7 +64,7 @@ public ResponseEntity> checkHealth() { Map healthStatus = healthService.checkHealth(); String overallStatus = (String) healthStatus.get("status"); - HttpStatus httpStatus = "UP".equals(overallStatus) ? HttpStatus.OK : HttpStatus.SERVICE_UNAVAILABLE; + HttpStatus httpStatus = "DOWN".equals(overallStatus) ? HttpStatus.SERVICE_UNAVAILABLE : HttpStatus.OK; logger.debug("Health check completed with status: {}", overallStatus); return new ResponseEntity<>(healthStatus, httpStatus); From 445b6cc1e56a6fe708f87ee7ab593aba6984ff3f Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Thu, 19 Feb 2026 21:35:17 +0530 Subject: [PATCH 04/14] fix(health): run checks concurrently, prevent thread starvation, and harden timeouts --- .../service/health/HealthService.java | 136 +++++++++++------- 1 file changed, 82 insertions(+), 54 deletions(-) diff --git a/src/main/java/com/iemr/inventory/service/health/HealthService.java b/src/main/java/com/iemr/inventory/service/health/HealthService.java index 2d4ec30e..fc3f5969 100644 --- a/src/main/java/com/iemr/inventory/service/health/HealthService.java +++ b/src/main/java/com/iemr/inventory/service/health/HealthService.java @@ -98,16 +98,35 @@ public Map checkHealth() { Map response = new LinkedHashMap<>(); response.put("timestamp", Instant.now().toString()); - Map> components = new LinkedHashMap<>(); - - // Check MySQL Map mysqlStatus = new LinkedHashMap<>(); - performHealthCheck("MySQL", mysqlStatus, this::checkMySQLHealth); - components.put("mysql", mysqlStatus); - - // Check Redis Map redisStatus = new LinkedHashMap<>(); - performHealthCheck("Redis", redisStatus, this::checkRedisHealth); + + // Submit both checks concurrently + CompletableFuture mysqlFuture = CompletableFuture.runAsync( + () -> performHealthCheck("MySQL", mysqlStatus, this::checkMySQLHealthSync), executorService); + CompletableFuture redisFuture = CompletableFuture.runAsync( + () -> performHealthCheck("Redis", redisStatus, this::checkRedisHealthSync), executorService); + + // Wait for both checks to complete with combined timeout + long maxTimeout = Math.max(MYSQL_TIMEOUT_SECONDS, REDIS_TIMEOUT_SECONDS) + 1; + try { + CompletableFuture.allOf(mysqlFuture, redisFuture) + .get(maxTimeout, TimeUnit.SECONDS); + } catch (TimeoutException e) { + logger.warn("Health check aggregate timeout after {} seconds", maxTimeout); + mysqlFuture.cancel(true); + redisFuture.cancel(true); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("Health check was interrupted"); + mysqlFuture.cancel(true); + redisFuture.cancel(true); + } catch (Exception e) { + logger.warn("Health check execution error: {}", e.getMessage()); + } + + Map> components = new LinkedHashMap<>(); + components.put("mysql", mysqlStatus); components.put("redis", redisStatus); response.put("components", components); @@ -119,29 +138,54 @@ public Map checkHealth() { return response; } - private HealthCheckResult checkMySQLHealth() { - CompletableFuture future = CompletableFuture.supplyAsync(() -> { - try (Connection connection = dataSource.getConnection(); - PreparedStatement stmt = connection.prepareStatement("SELECT 1 as health_check")) { - - stmt.setQueryTimeout((int) MYSQL_TIMEOUT_SECONDS); - - try (ResultSet rs = stmt.executeQuery()) { - if (rs.next()) { - return new HealthCheckResult(true, null); - } + private HealthCheckResult checkMySQLHealthSync() { + try (Connection connection = dataSource.getConnection(); + PreparedStatement stmt = connection.prepareStatement("SELECT 1 as health_check")) { + + stmt.setQueryTimeout((int) MYSQL_TIMEOUT_SECONDS); + + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + return new HealthCheckResult(true, null); } - - return new HealthCheckResult(false, "No result from health check query"); - - } catch (Exception e) { - logger.warn("MySQL health check failed: {}", e.getMessage(), e); - return new HealthCheckResult(false, "MySQL connection failed"); } - }, executorService); + + return new HealthCheckResult(false, "No result from health check query"); + + } catch (Exception e) { + logger.warn("MySQL health check failed: {}", e.getMessage(), e); + return new HealthCheckResult(false, "MySQL connection failed"); + } + } + + private HealthCheckResult checkRedisHealthSync() { + if (redisTemplate == null) { + return new HealthCheckResult(true, "Redis not configured — skipped"); + } try { - return future.get(MYSQL_TIMEOUT_SECONDS, TimeUnit.SECONDS); + String pong = redisTemplate.execute((org.springframework.data.redis.core.RedisCallback) (connection) -> connection.ping()); + + if ("PONG".equals(pong)) { + return new HealthCheckResult(true, null); + } + + return new HealthCheckResult(false, "Redis PING failed"); + + } catch (Exception e) { + logger.warn("Redis health check failed: {}", e.getMessage(), e); + return new HealthCheckResult(false, "Redis connection failed"); + } + } + + // Deprecated: kept for backward compatibility, use synchronous versions instead + @Deprecated + private HealthCheckResult checkMySQLHealth() { + CompletableFuture future = CompletableFuture.supplyAsync( + this::checkMySQLHealthSync, executorService); + + try { + return future.get(MYSQL_TIMEOUT_SECONDS + 1, TimeUnit.SECONDS); } catch (TimeoutException e) { future.cancel(true); logger.warn("MySQL health check timed out after {} seconds", MYSQL_TIMEOUT_SECONDS); @@ -156,39 +200,20 @@ private HealthCheckResult checkMySQLHealth() { } } + @Deprecated private HealthCheckResult checkRedisHealth() { - if (redisTemplate == null) { - return new HealthCheckResult(true, "Redis not configured — skipped"); - } + CompletableFuture future = CompletableFuture.supplyAsync( + this::checkRedisHealthSync, executorService); - CompletableFuture future = null; try { - future = CompletableFuture.supplyAsync(() -> { - try { - return redisTemplate.execute((org.springframework.data.redis.core.RedisCallback) (connection) -> connection.ping()); - } catch (Exception e) { - logger.debug("Redis PING failed: {}", e.getMessage(), e); - return null; - } - }, executorService); - - String pong = future.get(REDIS_TIMEOUT_SECONDS, TimeUnit.SECONDS); - - if ("PONG".equals(pong)) { - return new HealthCheckResult(true, null); - } - - return new HealthCheckResult(false, "Redis PING failed"); - + return future.get(REDIS_TIMEOUT_SECONDS + 1, TimeUnit.SECONDS); } catch (TimeoutException e) { - if (future != null) { - future.cancel(true); - } + future.cancel(true); logger.warn("Redis health check timed out after {} seconds", REDIS_TIMEOUT_SECONDS); return new HealthCheckResult(false, "Redis health check timed out"); } catch (InterruptedException e) { - logger.warn("Redis health check was interrupted"); Thread.currentThread().interrupt(); + logger.warn("Redis health check was interrupted"); return new HealthCheckResult(false, "Redis health check was interrupted"); } catch (Exception e) { logger.warn("Redis health check failed: {}", e.getMessage(), e); @@ -215,9 +240,12 @@ private Map performHealthCheck(String componentName, String severity = determineSeverity(result.isHealthy, responseTime); status.put(SEVERITY_KEY, severity); - // Include error message if present (sanitized) + // Include message or error based on health status if (result.error != null) { - status.put("error", result.error); + // Use "message" key for informational messages when healthy + // Use "error" key for actual error messages when unhealthy + String fieldKey = result.isHealthy ? "message" : "error"; + status.put(fieldKey, result.error); } return status; From bde74f16aa2f5ed82e7035a5b936ccf7425620e5 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Thu, 19 Feb 2026 22:00:40 +0530 Subject: [PATCH 05/14] fix(health): add proper @Deprecated metadata and javadoc for obsolete methods --- .../iemr/inventory/service/health/HealthService.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/iemr/inventory/service/health/HealthService.java b/src/main/java/com/iemr/inventory/service/health/HealthService.java index fc3f5969..b571fa08 100644 --- a/src/main/java/com/iemr/inventory/service/health/HealthService.java +++ b/src/main/java/com/iemr/inventory/service/health/HealthService.java @@ -178,8 +178,7 @@ private HealthCheckResult checkRedisHealthSync() { } } - // Deprecated: kept for backward compatibility, use synchronous versions instead - @Deprecated + @Deprecated(since = "3.6.0", forRemoval = true) private HealthCheckResult checkMySQLHealth() { CompletableFuture future = CompletableFuture.supplyAsync( this::checkMySQLHealthSync, executorService); @@ -200,7 +199,7 @@ private HealthCheckResult checkMySQLHealth() { } } - @Deprecated + @Deprecated(since = "3.6.0", forRemoval = true) private HealthCheckResult checkRedisHealth() { CompletableFuture future = CompletableFuture.supplyAsync( this::checkRedisHealthSync, executorService); @@ -263,13 +262,6 @@ private Map performHealthCheck(String componentName, } } - /** - * Determine severity level based on component health and response time. - * - * @param isHealthy true if component is healthy - * @param responseTimeMs response time in milliseconds - * @return severity level: OK, WARNING, or CRITICAL - */ private String determineSeverity(boolean isHealthy, long responseTimeMs) { if (!isHealthy) { return SEVERITY_CRITICAL; From 3eccb0645c3b46c0742198c8de5baeb05ea46189 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Thu, 19 Feb 2026 22:03:40 +0530 Subject: [PATCH 06/14] fix(health): add proper @Deprecated metadata and javadoc for obsolete methods --- .../iemr/inventory/service/health/HealthService.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/com/iemr/inventory/service/health/HealthService.java b/src/main/java/com/iemr/inventory/service/health/HealthService.java index b571fa08..8d3b6e20 100644 --- a/src/main/java/com/iemr/inventory/service/health/HealthService.java +++ b/src/main/java/com/iemr/inventory/service/health/HealthService.java @@ -178,6 +178,11 @@ private HealthCheckResult checkRedisHealthSync() { } } + /** + * Deprecated: kept for backward compatibility, use synchronous versions instead. + * + * @deprecated Use {@link #checkMySQLHealthSync()} instead. + */ @Deprecated(since = "3.6.0", forRemoval = true) private HealthCheckResult checkMySQLHealth() { CompletableFuture future = CompletableFuture.supplyAsync( @@ -199,6 +204,11 @@ private HealthCheckResult checkMySQLHealth() { } } + /** + * Deprecated: kept for backward compatibility, use synchronous versions instead. + * + * @deprecated Use {@link #checkRedisHealthSync()} instead. + */ @Deprecated(since = "3.6.0", forRemoval = true) private HealthCheckResult checkRedisHealth() { CompletableFuture future = CompletableFuture.supplyAsync( From 1d6650edf4ded0d0487f9537724f100bd43bfa38 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Thu, 19 Feb 2026 22:07:54 +0530 Subject: [PATCH 07/14] refactor(health): remove obsolete deprecated health check methods --- .../iemr/inventory/service/health/HealthService.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iemr/inventory/service/health/HealthService.java b/src/main/java/com/iemr/inventory/service/health/HealthService.java index 8d3b6e20..a7294c3a 100644 --- a/src/main/java/com/iemr/inventory/service/health/HealthService.java +++ b/src/main/java/com/iemr/inventory/service/health/HealthService.java @@ -180,8 +180,11 @@ private HealthCheckResult checkRedisHealthSync() { /** * Deprecated: kept for backward compatibility, use synchronous versions instead. + *

+ * This method will be removed in version 4.0.0. Use {@link #checkMySQLHealthSync()} instead. + *

* - * @deprecated Use {@link #checkMySQLHealthSync()} instead. + * @deprecated Since version 3.6.0. Scheduled for removal in version 4.0.0. */ @Deprecated(since = "3.6.0", forRemoval = true) private HealthCheckResult checkMySQLHealth() { @@ -206,8 +209,11 @@ private HealthCheckResult checkMySQLHealth() { /** * Deprecated: kept for backward compatibility, use synchronous versions instead. + *

+ * This method will be removed in version 4.0.0. Use {@link #checkRedisHealthSync()} instead. + *

* - * @deprecated Use {@link #checkRedisHealthSync()} instead. + * @deprecated Since version 3.6.0. Scheduled for removal in version 4.0.0. */ @Deprecated(since = "3.6.0", forRemoval = true) private HealthCheckResult checkRedisHealth() { From d0031c708d9b461d09c5c6374bd1b72307fcdae6 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Thu, 19 Feb 2026 22:14:49 +0530 Subject: [PATCH 08/14] fix(health): mark timed-out components DOWN and make status maps thread-safe --- .../iemr/inventory/service/health/HealthService.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/java/com/iemr/inventory/service/health/HealthService.java b/src/main/java/com/iemr/inventory/service/health/HealthService.java index a7294c3a..495aa146 100644 --- a/src/main/java/com/iemr/inventory/service/health/HealthService.java +++ b/src/main/java/com/iemr/inventory/service/health/HealthService.java @@ -125,6 +125,10 @@ public Map checkHealth() { logger.warn("Health check execution error: {}", e.getMessage()); } + // Ensure timed-out or unfinished components are marked DOWN + ensurePopulated(mysqlStatus, "MySQL"); + ensurePopulated(redisStatus, "Redis"); + Map> components = new LinkedHashMap<>(); components.put("mysql", mysqlStatus); components.put("redis", redisStatus); @@ -138,6 +142,14 @@ public Map checkHealth() { return response; } + private void ensurePopulated(Map status, String componentName) { + if (!status.containsKey(STATUS_KEY)) { + status.put(STATUS_KEY, STATUS_DOWN); + status.put(SEVERITY_KEY, SEVERITY_CRITICAL); + status.put("error", componentName + " health check did not complete in time"); + } + } + private HealthCheckResult checkMySQLHealthSync() { try (Connection connection = dataSource.getConnection(); PreparedStatement stmt = connection.prepareStatement("SELECT 1 as health_check")) { From e826dde083da7395a59846c72cdc3bf9fa2b9545 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Thu, 19 Feb 2026 22:21:25 +0530 Subject: [PATCH 09/14] fix(health): removed duplicates from health services --- .../service/health/HealthService.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/iemr/inventory/service/health/HealthService.java b/src/main/java/com/iemr/inventory/service/health/HealthService.java index 495aa146..de4c3e9f 100644 --- a/src/main/java/com/iemr/inventory/service/health/HealthService.java +++ b/src/main/java/com/iemr/inventory/service/health/HealthService.java @@ -59,6 +59,11 @@ public class HealthService { private static final String SEVERITY_WARNING = "WARNING"; private static final String SEVERITY_CRITICAL = "CRITICAL"; + // Response keys + private static final String ERROR_KEY = "error"; + private static final String MESSAGE_KEY = "message"; + private static final String RESPONSE_TIME_KEY = "responseTimeMs"; + // Timeouts (in seconds) private static final long MYSQL_TIMEOUT_SECONDS = 3; private static final long REDIS_TIMEOUT_SECONDS = 3; @@ -146,7 +151,7 @@ private void ensurePopulated(Map status, String componentName) { if (!status.containsKey(STATUS_KEY)) { status.put(STATUS_KEY, STATUS_DOWN); status.put(SEVERITY_KEY, SEVERITY_CRITICAL); - status.put("error", componentName + " health check did not complete in time"); + status.put(ERROR_KEY, componentName + " health check did not complete in time"); } } @@ -261,7 +266,7 @@ private Map performHealthCheck(String componentName, status.put(STATUS_KEY, result.isHealthy ? STATUS_UP : STATUS_DOWN); // Set response time - status.put("responseTimeMs", responseTime); + status.put(RESPONSE_TIME_KEY, responseTime); // Determine severity based on response time and status String severity = determineSeverity(result.isHealthy, responseTime); @@ -269,9 +274,9 @@ private Map performHealthCheck(String componentName, // Include message or error based on health status if (result.error != null) { - // Use "message" key for informational messages when healthy - // Use "error" key for actual error messages when unhealthy - String fieldKey = result.isHealthy ? "message" : "error"; + // Use MESSAGE_KEY for informational messages when healthy + // Use ERROR_KEY for actual error messages when unhealthy + String fieldKey = result.isHealthy ? MESSAGE_KEY : ERROR_KEY; status.put(fieldKey, result.error); } @@ -282,9 +287,9 @@ private Map performHealthCheck(String componentName, logger.error("{} health check failed with exception: {}", componentName, e.getMessage(), e); status.put(STATUS_KEY, STATUS_DOWN); - status.put("responseTimeMs", responseTime); + status.put(RESPONSE_TIME_KEY, responseTime); status.put(SEVERITY_KEY, SEVERITY_CRITICAL); - status.put("error", "Health check failed with an unexpected error"); + status.put(ERROR_KEY, "Health check failed with an unexpected error"); return status; } From 9ea2afca90cfd51a3cb0c2dcc753786354f302fc Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Fri, 20 Feb 2026 20:53:43 +0530 Subject: [PATCH 10/14] fix(health): harden advanced MySQL checks and throttle execution --- .../controller/health/HealthController.java | 3 +- .../service/health/HealthService.java | 285 +++++++++++++++--- 2 files changed, 252 insertions(+), 36 deletions(-) diff --git a/src/main/java/com/iemr/inventory/controller/health/HealthController.java b/src/main/java/com/iemr/inventory/controller/health/HealthController.java index a085d04a..59fb160b 100644 --- a/src/main/java/com/iemr/inventory/controller/health/HealthController.java +++ b/src/main/java/com/iemr/inventory/controller/health/HealthController.java @@ -54,7 +54,7 @@ public HealthController(HealthService healthService) { @Operation(summary = "Check infrastructure health", description = "Returns the health status of MySQL, Redis, and other configured services") @ApiResponses({ - @ApiResponse(responseCode = "200", description = "All checked components are UP"), + @ApiResponse(responseCode = "200", description = "Services are UP or DEGRADED (operational with warnings)"), @ApiResponse(responseCode = "503", description = "One or more critical services are DOWN") }) public ResponseEntity> checkHealth() { @@ -64,6 +64,7 @@ public ResponseEntity> checkHealth() { Map healthStatus = healthService.checkHealth(); String overallStatus = (String) healthStatus.get("status"); + // Return 503 only if DOWN; 200 for both UP and DEGRADED (DEGRADED = operational with warnings) HttpStatus httpStatus = "DOWN".equals(overallStatus) ? HttpStatus.SERVICE_UNAVAILABLE : HttpStatus.OK; logger.debug("Health check completed with status: {}", overallStatus); diff --git a/src/main/java/com/iemr/inventory/service/health/HealthService.java b/src/main/java/com/iemr/inventory/service/health/HealthService.java index de4c3e9f..931f4504 100644 --- a/src/main/java/com/iemr/inventory/service/health/HealthService.java +++ b/src/main/java/com/iemr/inventory/service/health/HealthService.java @@ -36,9 +36,16 @@ import java.util.function.Supplier; import jakarta.annotation.PreDestroy; import javax.sql.DataSource; +import com.zaxxer.hikari.HikariDataSource; +import com.zaxxer.hikari.HikariPoolMXBean; +import java.lang.management.ManagementFactory; +import javax.management.MBeanServer; +import javax.management.ObjectName; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; @@ -68,12 +75,30 @@ public class HealthService { private static final long MYSQL_TIMEOUT_SECONDS = 3; private static final long REDIS_TIMEOUT_SECONDS = 3; + // Advanced checks configuration + private static final long ADVANCED_CHECKS_TIMEOUT_MS = 500; // Strict timeout for advanced checks + private static final long ADVANCED_CHECKS_THROTTLE_SECONDS = 30; // Run at most once per 30 seconds + // Performance threshold (milliseconds) - response time > 2000ms = DEGRADED private static final long RESPONSE_TIME_THRESHOLD_MS = 2000; + + // Diagnostic event codes for concise logging + private static final String DIAGNOSTIC_LOCK_WAIT = "MYSQL_LOCK_WAIT"; + private static final String DIAGNOSTIC_DEADLOCK = "MYSQL_DEADLOCK"; + private static final String DIAGNOSTIC_SLOW_QUERIES = "MYSQL_SLOW_QUERIES"; + private static final String DIAGNOSTIC_POOL_EXHAUSTED = "MYSQL_POOL_EXHAUSTED"; private final DataSource dataSource; private final RedisTemplate redisTemplate; private final ExecutorService executorService; + + // Advanced checks throttling (thread-safe) + private volatile long lastAdvancedCheckTime = 0; + private volatile AdvancedCheckResult cachedAdvancedCheckResult = null; + private final ReentrantReadWriteLock advancedCheckLock = new ReentrantReadWriteLock(); + + @Value("${health.advanced.enabled:true}") + private boolean advancedHealthChecksEnabled; public HealthService(DataSource dataSource, @Autowired(required = false) RedisTemplate redisTemplate) { @@ -163,47 +188,40 @@ private HealthCheckResult checkMySQLHealthSync() { try (ResultSet rs = stmt.executeQuery()) { if (rs.next()) { - return new HealthCheckResult(true, null); + // Basic health check passed, now run advanced checks with throttling + boolean isDegraded = performAdvancedMySQLChecksWithThrottle(connection); + return new HealthCheckResult(true, null, isDegraded); } } - return new HealthCheckResult(false, "No result from health check query"); + return new HealthCheckResult(false, "No result from health check query", false); } catch (Exception e) { logger.warn("MySQL health check failed: {}", e.getMessage(), e); - return new HealthCheckResult(false, "MySQL connection failed"); + return new HealthCheckResult(false, "MySQL connection failed", false); } } private HealthCheckResult checkRedisHealthSync() { if (redisTemplate == null) { - return new HealthCheckResult(true, "Redis not configured — skipped"); + return new HealthCheckResult(true, "Redis not configured — skipped", false); } try { String pong = redisTemplate.execute((org.springframework.data.redis.core.RedisCallback) (connection) -> connection.ping()); if ("PONG".equals(pong)) { - return new HealthCheckResult(true, null); + return new HealthCheckResult(true, null, false); } - return new HealthCheckResult(false, "Redis PING failed"); + return new HealthCheckResult(false, "Redis PING failed", false); } catch (Exception e) { logger.warn("Redis health check failed: {}", e.getMessage(), e); - return new HealthCheckResult(false, "Redis connection failed"); + return new HealthCheckResult(false, "Redis connection failed", false); } } - /** - * Deprecated: kept for backward compatibility, use synchronous versions instead. - *

- * This method will be removed in version 4.0.0. Use {@link #checkMySQLHealthSync()} instead. - *

- * - * @deprecated Since version 3.6.0. Scheduled for removal in version 4.0.0. - */ - @Deprecated(since = "3.6.0", forRemoval = true) private HealthCheckResult checkMySQLHealth() { CompletableFuture future = CompletableFuture.supplyAsync( this::checkMySQLHealthSync, executorService); @@ -213,26 +231,17 @@ private HealthCheckResult checkMySQLHealth() { } catch (TimeoutException e) { future.cancel(true); logger.warn("MySQL health check timed out after {} seconds", MYSQL_TIMEOUT_SECONDS); - return new HealthCheckResult(false, "MySQL health check timed out"); + return new HealthCheckResult(false, "MySQL health check timed out", false); } catch (InterruptedException e) { Thread.currentThread().interrupt(); logger.warn("MySQL health check was interrupted"); - return new HealthCheckResult(false, "MySQL health check was interrupted"); + return new HealthCheckResult(false, "MySQL health check was interrupted", false); } catch (Exception e) { logger.warn("MySQL health check failed: {}", e.getMessage(), e); - return new HealthCheckResult(false, "MySQL connection failed"); + return new HealthCheckResult(false, "MySQL connection failed", false); } } - /** - * Deprecated: kept for backward compatibility, use synchronous versions instead. - *

- * This method will be removed in version 4.0.0. Use {@link #checkRedisHealthSync()} instead. - *

- * - * @deprecated Since version 3.6.0. Scheduled for removal in version 4.0.0. - */ - @Deprecated(since = "3.6.0", forRemoval = true) private HealthCheckResult checkRedisHealth() { CompletableFuture future = CompletableFuture.supplyAsync( this::checkRedisHealthSync, executorService); @@ -242,14 +251,14 @@ private HealthCheckResult checkRedisHealth() { } catch (TimeoutException e) { future.cancel(true); logger.warn("Redis health check timed out after {} seconds", REDIS_TIMEOUT_SECONDS); - return new HealthCheckResult(false, "Redis health check timed out"); + return new HealthCheckResult(false, "Redis health check timed out", false); } catch (InterruptedException e) { Thread.currentThread().interrupt(); logger.warn("Redis health check was interrupted"); - return new HealthCheckResult(false, "Redis health check was interrupted"); + return new HealthCheckResult(false, "Redis health check was interrupted", false); } catch (Exception e) { logger.warn("Redis health check failed: {}", e.getMessage(), e); - return new HealthCheckResult(false, "Redis connection failed"); + return new HealthCheckResult(false, "Redis connection failed", false); } } @@ -268,8 +277,8 @@ private Map performHealthCheck(String componentName, // Set response time status.put(RESPONSE_TIME_KEY, responseTime); - // Determine severity based on response time and status - String severity = determineSeverity(result.isHealthy, responseTime); + // Determine severity based on health, response time, and degradation flags + String severity = determineSeverity(result.isHealthy, responseTime, result.isDegraded); status.put(SEVERITY_KEY, severity); // Include message or error based on health status @@ -295,11 +304,15 @@ private Map performHealthCheck(String componentName, } } - private String determineSeverity(boolean isHealthy, long responseTimeMs) { + private String determineSeverity(boolean isHealthy, long responseTimeMs, boolean isDegraded) { if (!isHealthy) { return SEVERITY_CRITICAL; } + if (isDegraded) { + return SEVERITY_WARNING; + } + if (responseTimeMs > RESPONSE_TIME_THRESHOLD_MS) { return SEVERITY_WARNING; } @@ -335,13 +348,215 @@ private String computeOverallStatus(Map> components) return STATUS_UP; } + // Internal advanced health checks for MySQL - do not expose details in responses + private boolean performAdvancedMySQLChecksWithThrottle(Connection connection) { + if (!advancedHealthChecksEnabled) { + return false; // Advanced checks disabled + } + + long currentTime = System.currentTimeMillis(); + + // Check throttle window - use read lock first for fast path + advancedCheckLock.readLock().lock(); + try { + if (cachedAdvancedCheckResult != null && + (currentTime - lastAdvancedCheckTime) < ADVANCED_CHECKS_THROTTLE_SECONDS * 1000) { + // Return cached result - within throttle window + return cachedAdvancedCheckResult.isDegraded; + } + } finally { + advancedCheckLock.readLock().unlock(); + } + + // Outside throttle window - acquire write lock and run checks + advancedCheckLock.writeLock().lock(); + try { + // Double-check after acquiring write lock + if (cachedAdvancedCheckResult != null && + (currentTime - lastAdvancedCheckTime) < ADVANCED_CHECKS_THROTTLE_SECONDS * 1000) { + return cachedAdvancedCheckResult.isDegraded; + } + + // Execute advanced checks with strict timeout + AdvancedCheckResult result = executeAdvancedChecksWithTimeout(connection); + + // Cache the result + lastAdvancedCheckTime = currentTime; + cachedAdvancedCheckResult = result; + + return result.isDegraded; + } finally { + advancedCheckLock.writeLock().unlock(); + } + } + + private AdvancedCheckResult executeAdvancedChecksWithTimeout(Connection connection) { + CompletableFuture checksFuture = CompletableFuture.supplyAsync( + () -> performAdvancedMySQLChecks(connection), executorService); + + try { + return checksFuture.get(ADVANCED_CHECKS_TIMEOUT_MS, java.util.concurrent.TimeUnit.MILLISECONDS); + } catch (java.util.concurrent.TimeoutException e) { + checksFuture.cancel(true); + logger.warn("Advanced MySQL diagnostics timed out ({}ms)", ADVANCED_CHECKS_TIMEOUT_MS); + return new AdvancedCheckResult(true); // Mark as degraded on timeout + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("Advanced MySQL diagnostics interrupted"); + return new AdvancedCheckResult(true); + } catch (Exception e) { + logger.warn("Advanced MySQL diagnostics execution failed"); + return new AdvancedCheckResult(true); + } + } + + private AdvancedCheckResult performAdvancedMySQLChecks(Connection connection) { + try { + boolean hasIssues = false; + + if (hasLockWaits(connection)) { + logger.warn("Diagnostic: {}", DIAGNOSTIC_LOCK_WAIT); + hasIssues = true; + } + + if (hasDeadlocks(connection)) { + logger.warn("Diagnostic: {}", DIAGNOSTIC_DEADLOCK); + hasIssues = true; + } + + if (hasSlowQueries(connection)) { + logger.warn("Diagnostic: {}", DIAGNOSTIC_SLOW_QUERIES); + hasIssues = true; + } + + if (hasConnectionPoolExhaustion()) { + logger.warn("Diagnostic: {}", DIAGNOSTIC_POOL_EXHAUSTED); + hasIssues = true; + } + + return new AdvancedCheckResult(hasIssues); + } catch (Exception e) { + logger.debug("Advanced MySQL checks encountered exception, marking degraded"); + return new AdvancedCheckResult(true); + } + } + + private boolean hasLockWaits(Connection connection) { + try (PreparedStatement stmt = connection.prepareStatement( + "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + + "WHERE state LIKE '%Lock%' " + + "AND user NOT IN ('event_scheduler', 'system user', 'root')")) { + stmt.setQueryTimeout(2); + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + int lockCount = rs.getInt(1); + return lockCount > 0; + } + } + } catch (Exception e) { + logger.debug("Could not check for lock waits"); + } + return false; + } + + private boolean hasDeadlocks(Connection connection) { + try (PreparedStatement stmt = connection.prepareStatement("SHOW ENGINE INNODB STATUS")) { + stmt.setQueryTimeout(2); + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + String innodbStatus = rs.getString(3); + return innodbStatus != null && innodbStatus.contains("LATEST DETECTED DEADLOCK"); + } + } + } catch (Exception e) { + logger.debug("Could not check for deadlocks"); + } + return false; + } + + private boolean hasSlowQueries(Connection connection) { + try (PreparedStatement stmt = connection.prepareStatement( + "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + + "WHERE command != 'Sleep' AND time > ? AND user NOT IN ('event_scheduler', 'system user')")) { + stmt.setQueryTimeout(2); + stmt.setInt(1, 10); // Queries running longer than 10 seconds + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + int slowQueryCount = rs.getInt(1); + return slowQueryCount > 3; // Alert if more than 3 slow queries + } + } + } catch (Exception e) { + logger.debug("Could not check for slow queries"); + } + return false; + } + + private boolean hasConnectionPoolExhaustion() { + // Use HikariCP metrics if available + if (dataSource instanceof HikariDataSource) { + try { + HikariDataSource hikariDataSource = (HikariDataSource) dataSource; + HikariPoolMXBean poolMXBean = hikariDataSource.getHikariPoolMXBean(); + + if (poolMXBean != null) { + int activeConnections = poolMXBean.getActiveConnections(); + int maxPoolSize = hikariDataSource.getMaximumPoolSize(); + + // Alert if > 80% of pool is exhausted + int threshold = (int) (maxPoolSize * 0.8); + return activeConnections > threshold; + } + } catch (Exception e) { + logger.debug("Could not retrieve HikariCP pool metrics"); + } + } + + // Fallback: try to get pool metrics via JMX if HikariCP is not directly available + try { + MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer(); + ObjectName objectName = new ObjectName("com.zaxxer.hikari:type=Pool (*)"); + var mBeans = mBeanServer.queryMBeans(objectName, null); + + for (var mBean : mBeans) { + try { + Integer activeConnections = (Integer) mBeanServer.getAttribute(mBean.getObjectName(), "ActiveConnections"); + Integer maximumPoolSize = (Integer) mBeanServer.getAttribute(mBean.getObjectName(), "MaximumPoolSize"); + + if (activeConnections != null && maximumPoolSize != null) { + int threshold = (int) (maximumPoolSize * 0.8); + return activeConnections > threshold; + } + } catch (Exception e) { + // Continue to next MBean + } + } + } catch (Exception e) { + logger.debug("Could not access HikariCP pool metrics via JMX"); + } + + // No pool metrics available - disable this check + logger.debug("Pool exhaustion check disabled: HikariCP metrics unavailable"); + return false; + } + + private static class AdvancedCheckResult { + final boolean isDegraded; + + AdvancedCheckResult(boolean isDegraded) { + this.isDegraded = isDegraded; + } + } + private static class HealthCheckResult { final boolean isHealthy; final String error; + final boolean isDegraded; - HealthCheckResult(boolean isHealthy, String error) { + HealthCheckResult(boolean isHealthy, String error, boolean isDegraded) { this.isHealthy = isHealthy; this.error = error; + this.isDegraded = isDegraded; } } } From 9640050a70af3360d7d82e8ba4398d14f88b79b3 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Fri, 20 Feb 2026 21:07:04 +0530 Subject: [PATCH 11/14] fix(health): harden advanced MySQL checks and reflect DEGRADED status --- .../service/health/HealthService.java | 128 +++++++++--------- 1 file changed, 66 insertions(+), 62 deletions(-) diff --git a/src/main/java/com/iemr/inventory/service/health/HealthService.java b/src/main/java/com/iemr/inventory/service/health/HealthService.java index 931f4504..6c886fed 100644 --- a/src/main/java/com/iemr/inventory/service/health/HealthService.java +++ b/src/main/java/com/iemr/inventory/service/health/HealthService.java @@ -87,6 +87,7 @@ public class HealthService { private static final String DIAGNOSTIC_DEADLOCK = "MYSQL_DEADLOCK"; private static final String DIAGNOSTIC_SLOW_QUERIES = "MYSQL_SLOW_QUERIES"; private static final String DIAGNOSTIC_POOL_EXHAUSTED = "MYSQL_POOL_EXHAUSTED"; + private static final String DIAGNOSTIC_LOG_TEMPLATE = "Diagnostic: {}"; private final DataSource dataSource; private final RedisTemplate redisTemplate; @@ -97,6 +98,9 @@ public class HealthService { private volatile AdvancedCheckResult cachedAdvancedCheckResult = null; private final ReentrantReadWriteLock advancedCheckLock = new ReentrantReadWriteLock(); + // Deadlock check resilience - disable after first permission error + private volatile boolean deadlockCheckDisabled = false; + @Value("${health.advanced.enabled:true}") private boolean advancedHealthChecksEnabled; @@ -222,46 +226,6 @@ private HealthCheckResult checkRedisHealthSync() { } } - private HealthCheckResult checkMySQLHealth() { - CompletableFuture future = CompletableFuture.supplyAsync( - this::checkMySQLHealthSync, executorService); - - try { - return future.get(MYSQL_TIMEOUT_SECONDS + 1, TimeUnit.SECONDS); - } catch (TimeoutException e) { - future.cancel(true); - logger.warn("MySQL health check timed out after {} seconds", MYSQL_TIMEOUT_SECONDS); - return new HealthCheckResult(false, "MySQL health check timed out", false); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.warn("MySQL health check was interrupted"); - return new HealthCheckResult(false, "MySQL health check was interrupted", false); - } catch (Exception e) { - logger.warn("MySQL health check failed: {}", e.getMessage(), e); - return new HealthCheckResult(false, "MySQL connection failed", false); - } - } - - private HealthCheckResult checkRedisHealth() { - CompletableFuture future = CompletableFuture.supplyAsync( - this::checkRedisHealthSync, executorService); - - try { - return future.get(REDIS_TIMEOUT_SECONDS + 1, TimeUnit.SECONDS); - } catch (TimeoutException e) { - future.cancel(true); - logger.warn("Redis health check timed out after {} seconds", REDIS_TIMEOUT_SECONDS); - return new HealthCheckResult(false, "Redis health check timed out", false); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.warn("Redis health check was interrupted"); - return new HealthCheckResult(false, "Redis health check was interrupted", false); - } catch (Exception e) { - logger.warn("Redis health check failed: {}", e.getMessage(), e); - return new HealthCheckResult(false, "Redis connection failed", false); - } - } - private Map performHealthCheck(String componentName, Map status, Supplier checker) { @@ -271,8 +235,16 @@ private Map performHealthCheck(String componentName, HealthCheckResult result = checker.get(); long responseTime = System.currentTimeMillis() - startTime; - // Set status - status.put(STATUS_KEY, result.isHealthy ? STATUS_UP : STATUS_DOWN); + // Determine status: DOWN (unhealthy), DEGRADED (healthy but with issues), or UP + String componentStatus; + if (!result.isHealthy) { + componentStatus = STATUS_DOWN; + } else if (result.isDegraded) { + componentStatus = STATUS_DEGRADED; + } else { + componentStatus = STATUS_UP; + } + status.put(STATUS_KEY, componentStatus); // Set response time status.put(RESPONSE_TIME_KEY, responseTime); @@ -322,7 +294,7 @@ private String determineSeverity(boolean isHealthy, long responseTimeMs, boolean private String computeOverallStatus(Map> components) { boolean hasCritical = false; - boolean hasWarning = false; + boolean hasDegraded = false; for (Map componentStatus : components.values()) { String status = (String) componentStatus.get(STATUS_KEY); @@ -332,8 +304,12 @@ private String computeOverallStatus(Map> components) hasCritical = true; } + if (STATUS_DEGRADED.equals(status)) { + hasDegraded = true; + } + if (SEVERITY_WARNING.equals(severity)) { - hasWarning = true; + hasDegraded = true; } } @@ -341,7 +317,7 @@ private String computeOverallStatus(Map> components) return STATUS_DOWN; } - if (hasWarning) { + if (hasDegraded) { return STATUS_DEGRADED; } @@ -415,22 +391,22 @@ private AdvancedCheckResult performAdvancedMySQLChecks(Connection connection) { boolean hasIssues = false; if (hasLockWaits(connection)) { - logger.warn("Diagnostic: {}", DIAGNOSTIC_LOCK_WAIT); + logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_LOCK_WAIT); hasIssues = true; } if (hasDeadlocks(connection)) { - logger.warn("Diagnostic: {}", DIAGNOSTIC_DEADLOCK); + logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_DEADLOCK); hasIssues = true; } if (hasSlowQueries(connection)) { - logger.warn("Diagnostic: {}", DIAGNOSTIC_SLOW_QUERIES); + logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_SLOW_QUERIES); hasIssues = true; } if (hasConnectionPoolExhaustion()) { - logger.warn("Diagnostic: {}", DIAGNOSTIC_POOL_EXHAUSTED); + logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_POOL_EXHAUSTED); hasIssues = true; } @@ -444,7 +420,9 @@ private AdvancedCheckResult performAdvancedMySQLChecks(Connection connection) { private boolean hasLockWaits(Connection connection) { try (PreparedStatement stmt = connection.prepareStatement( "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + - "WHERE state LIKE '%Lock%' " + + "WHERE (state = 'Waiting for table metadata lock' " + + " OR state = 'Waiting for row lock' " + + " OR state = 'Waiting for lock') " + "AND user NOT IN ('event_scheduler', 'system user', 'root')")) { stmt.setQueryTimeout(2); try (ResultSet rs = stmt.executeQuery()) { @@ -460,6 +438,11 @@ private boolean hasLockWaits(Connection connection) { } private boolean hasDeadlocks(Connection connection) { + // Skip deadlock check if already disabled due to permissions + if (deadlockCheckDisabled) { + return false; + } + try (PreparedStatement stmt = connection.prepareStatement("SHOW ENGINE INNODB STATUS")) { stmt.setQueryTimeout(2); try (ResultSet rs = stmt.executeQuery()) { @@ -468,6 +451,17 @@ private boolean hasDeadlocks(Connection connection) { return innodbStatus != null && innodbStatus.contains("LATEST DETECTED DEADLOCK"); } } + } catch (java.sql.SQLException e) { + // Check if this is a permission error + if (e.getMessage() != null && + (e.getMessage().contains("Access denied") || + e.getMessage().contains("permission"))) { + // Disable this check permanently after first permission error + deadlockCheckDisabled = true; + logger.warn("Deadlock check disabled: Insufficient privileges"); + } else { + logger.debug("Could not check for deadlocks"); + } } catch (Exception e) { logger.debug("Could not check for deadlocks"); } @@ -494,9 +488,8 @@ private boolean hasSlowQueries(Connection connection) { private boolean hasConnectionPoolExhaustion() { // Use HikariCP metrics if available - if (dataSource instanceof HikariDataSource) { + if (dataSource instanceof HikariDataSource hikariDataSource) { try { - HikariDataSource hikariDataSource = (HikariDataSource) dataSource; HikariPoolMXBean poolMXBean = hikariDataSource.getHikariPoolMXBean(); if (poolMXBean != null) { @@ -513,22 +506,18 @@ private boolean hasConnectionPoolExhaustion() { } // Fallback: try to get pool metrics via JMX if HikariCP is not directly available + return checkPoolMetricsViaJMX(); + } + + private boolean checkPoolMetricsViaJMX() { try { MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer(); ObjectName objectName = new ObjectName("com.zaxxer.hikari:type=Pool (*)"); var mBeans = mBeanServer.queryMBeans(objectName, null); for (var mBean : mBeans) { - try { - Integer activeConnections = (Integer) mBeanServer.getAttribute(mBean.getObjectName(), "ActiveConnections"); - Integer maximumPoolSize = (Integer) mBeanServer.getAttribute(mBean.getObjectName(), "MaximumPoolSize"); - - if (activeConnections != null && maximumPoolSize != null) { - int threshold = (int) (maximumPoolSize * 0.8); - return activeConnections > threshold; - } - } catch (Exception e) { - // Continue to next MBean + if (evaluatePoolMetrics(mBeanServer, mBean.getObjectName())) { + return true; } } } catch (Exception e) { @@ -540,6 +529,21 @@ private boolean hasConnectionPoolExhaustion() { return false; } + private boolean evaluatePoolMetrics(MBeanServer mBeanServer, ObjectName objectName) { + try { + Integer activeConnections = (Integer) mBeanServer.getAttribute(objectName, "ActiveConnections"); + Integer maximumPoolSize = (Integer) mBeanServer.getAttribute(objectName, "MaximumPoolSize"); + + if (activeConnections != null && maximumPoolSize != null) { + int threshold = (int) (maximumPoolSize * 0.8); + return activeConnections > threshold; + } + } catch (Exception e) { + // Continue to next MBean + } + return false; + } + private static class AdvancedCheckResult { final boolean isDegraded; From 9e88705553dd73037c5381607b2f2eda9b4aeb81 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Fri, 20 Feb 2026 21:11:21 +0530 Subject: [PATCH 12/14] fix(health): avoid nested executor deadlock in advanced MySQL checks --- .../service/health/HealthService.java | 23 +------------------ 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/src/main/java/com/iemr/inventory/service/health/HealthService.java b/src/main/java/com/iemr/inventory/service/health/HealthService.java index 6c886fed..3da8445b 100644 --- a/src/main/java/com/iemr/inventory/service/health/HealthService.java +++ b/src/main/java/com/iemr/inventory/service/health/HealthService.java @@ -353,8 +353,7 @@ private boolean performAdvancedMySQLChecksWithThrottle(Connection connection) { return cachedAdvancedCheckResult.isDegraded; } - // Execute advanced checks with strict timeout - AdvancedCheckResult result = executeAdvancedChecksWithTimeout(connection); + AdvancedCheckResult result = performAdvancedMySQLChecks(connection); // Cache the result lastAdvancedCheckTime = currentTime; @@ -366,26 +365,6 @@ private boolean performAdvancedMySQLChecksWithThrottle(Connection connection) { } } - private AdvancedCheckResult executeAdvancedChecksWithTimeout(Connection connection) { - CompletableFuture checksFuture = CompletableFuture.supplyAsync( - () -> performAdvancedMySQLChecks(connection), executorService); - - try { - return checksFuture.get(ADVANCED_CHECKS_TIMEOUT_MS, java.util.concurrent.TimeUnit.MILLISECONDS); - } catch (java.util.concurrent.TimeoutException e) { - checksFuture.cancel(true); - logger.warn("Advanced MySQL diagnostics timed out ({}ms)", ADVANCED_CHECKS_TIMEOUT_MS); - return new AdvancedCheckResult(true); // Mark as degraded on timeout - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.warn("Advanced MySQL diagnostics interrupted"); - return new AdvancedCheckResult(true); - } catch (Exception e) { - logger.warn("Advanced MySQL diagnostics execution failed"); - return new AdvancedCheckResult(true); - } - } - private AdvancedCheckResult performAdvancedMySQLChecks(Connection connection) { try { boolean hasIssues = false; From 2f8471d2d38c4ce0c725f23f307a65d950525da2 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sat, 21 Feb 2026 12:22:14 +0530 Subject: [PATCH 13/14] fix(health): scope PROCESSLIST lock-wait check to application DB user --- .../service/health/HealthService.java | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/iemr/inventory/service/health/HealthService.java b/src/main/java/com/iemr/inventory/service/health/HealthService.java index 3da8445b..23629b9a 100644 --- a/src/main/java/com/iemr/inventory/service/health/HealthService.java +++ b/src/main/java/com/iemr/inventory/service/health/HealthService.java @@ -28,9 +28,10 @@ import java.time.Instant; import java.util.LinkedHashMap; import java.util.Map; -import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.function.Supplier; @@ -132,20 +133,26 @@ public Map checkHealth() { Map response = new LinkedHashMap<>(); response.put("timestamp", Instant.now().toString()); - Map mysqlStatus = new LinkedHashMap<>(); - Map redisStatus = new LinkedHashMap<>(); + Map mysqlStatus = new ConcurrentHashMap<>(); + Map redisStatus = new ConcurrentHashMap<>(); - // Submit both checks concurrently - CompletableFuture mysqlFuture = CompletableFuture.runAsync( - () -> performHealthCheck("MySQL", mysqlStatus, this::checkMySQLHealthSync), executorService); - CompletableFuture redisFuture = CompletableFuture.runAsync( - () -> performHealthCheck("Redis", redisStatus, this::checkRedisHealthSync), executorService); + // Submit both checks concurrently using executorService for proper cancellation support + Future mysqlFuture = executorService.submit( + () -> performHealthCheck("MySQL", mysqlStatus, this::checkMySQLHealthSync)); + Future redisFuture = executorService.submit( + () -> performHealthCheck("Redis", redisStatus, this::checkRedisHealthSync)); - // Wait for both checks to complete with combined timeout + // Wait for both checks to complete with combined timeout (shared deadline) long maxTimeout = Math.max(MYSQL_TIMEOUT_SECONDS, REDIS_TIMEOUT_SECONDS) + 1; + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(maxTimeout); try { - CompletableFuture.allOf(mysqlFuture, redisFuture) - .get(maxTimeout, TimeUnit.SECONDS); + mysqlFuture.get(maxTimeout, TimeUnit.SECONDS); + long remainingNs = deadlineNs - System.nanoTime(); + if (remainingNs > 0) { + redisFuture.get(remainingNs, TimeUnit.NANOSECONDS); + } else { + redisFuture.cancel(true); + } } catch (TimeoutException e) { logger.warn("Health check aggregate timeout after {} seconds", maxTimeout); mysqlFuture.cancel(true); @@ -402,7 +409,7 @@ private boolean hasLockWaits(Connection connection) { "WHERE (state = 'Waiting for table metadata lock' " + " OR state = 'Waiting for row lock' " + " OR state = 'Waiting for lock') " + - "AND user NOT IN ('event_scheduler', 'system user', 'root')")) { + "AND user = USER()")) { stmt.setQueryTimeout(2); try (ResultSet rs = stmt.executeQuery()) { if (rs.next()) { From db1d3ba34af0dfa82a1d48b46a5b6953a807d171 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sun, 22 Feb 2026 21:08:11 +0530 Subject: [PATCH 14/14] fix(health): avoid blocking DB I/O under write lock and restore interrupt flag --- .../service/health/HealthService.java | 166 ++++++++---------- 1 file changed, 78 insertions(+), 88 deletions(-) diff --git a/src/main/java/com/iemr/inventory/service/health/HealthService.java b/src/main/java/com/iemr/inventory/service/health/HealthService.java index 23629b9a..3583b00b 100644 --- a/src/main/java/com/iemr/inventory/service/health/HealthService.java +++ b/src/main/java/com/iemr/inventory/service/health/HealthService.java @@ -34,6 +34,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.ExecutionException; import java.util.function.Supplier; import jakarta.annotation.PreDestroy; import javax.sql.DataSource; @@ -46,7 +47,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; @@ -85,7 +85,6 @@ public class HealthService { // Diagnostic event codes for concise logging private static final String DIAGNOSTIC_LOCK_WAIT = "MYSQL_LOCK_WAIT"; - private static final String DIAGNOSTIC_DEADLOCK = "MYSQL_DEADLOCK"; private static final String DIAGNOSTIC_SLOW_QUERIES = "MYSQL_SLOW_QUERIES"; private static final String DIAGNOSTIC_POOL_EXHAUSTED = "MYSQL_POOL_EXHAUSTED"; private static final String DIAGNOSTIC_LOG_TEMPLATE = "Diagnostic: {}"; @@ -99,17 +98,14 @@ public class HealthService { private volatile AdvancedCheckResult cachedAdvancedCheckResult = null; private final ReentrantReadWriteLock advancedCheckLock = new ReentrantReadWriteLock(); - // Deadlock check resilience - disable after first permission error - private volatile boolean deadlockCheckDisabled = false; - - @Value("${health.advanced.enabled:true}") - private boolean advancedHealthChecksEnabled; + // Advanced checks always enabled + private static final boolean ADVANCED_HEALTH_CHECKS_ENABLED = true; public HealthService(DataSource dataSource, @Autowired(required = false) RedisTemplate redisTemplate) { this.dataSource = dataSource; this.redisTemplate = redisTemplate; - this.executorService = Executors.newFixedThreadPool(2); + this.executorService = Executors.newFixedThreadPool(6); } @PreDestroy @@ -136,37 +132,10 @@ public Map checkHealth() { Map mysqlStatus = new ConcurrentHashMap<>(); Map redisStatus = new ConcurrentHashMap<>(); - // Submit both checks concurrently using executorService for proper cancellation support - Future mysqlFuture = executorService.submit( - () -> performHealthCheck("MySQL", mysqlStatus, this::checkMySQLHealthSync)); - Future redisFuture = executorService.submit( - () -> performHealthCheck("Redis", redisStatus, this::checkRedisHealthSync)); - - // Wait for both checks to complete with combined timeout (shared deadline) - long maxTimeout = Math.max(MYSQL_TIMEOUT_SECONDS, REDIS_TIMEOUT_SECONDS) + 1; - long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(maxTimeout); - try { - mysqlFuture.get(maxTimeout, TimeUnit.SECONDS); - long remainingNs = deadlineNs - System.nanoTime(); - if (remainingNs > 0) { - redisFuture.get(remainingNs, TimeUnit.NANOSECONDS); - } else { - redisFuture.cancel(true); - } - } catch (TimeoutException e) { - logger.warn("Health check aggregate timeout after {} seconds", maxTimeout); - mysqlFuture.cancel(true); - redisFuture.cancel(true); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.warn("Health check was interrupted"); - mysqlFuture.cancel(true); - redisFuture.cancel(true); - } catch (Exception e) { - logger.warn("Health check execution error: {}", e.getMessage()); + if (!executorService.isShutdown()) { + performHealthChecks(mysqlStatus, redisStatus); } - // Ensure timed-out or unfinished components are marked DOWN ensurePopulated(mysqlStatus, "MySQL"); ensurePopulated(redisStatus, "Redis"); @@ -175,14 +144,56 @@ public Map checkHealth() { components.put("redis", redisStatus); response.put("components", components); - - // Compute overall status - String overallStatus = computeOverallStatus(components); - response.put(STATUS_KEY, overallStatus); + response.put(STATUS_KEY, computeOverallStatus(components)); return response; } + private void performHealthChecks(Map mysqlStatus, Map redisStatus) { + Future mysqlFuture = null; + Future redisFuture = null; + try { + mysqlFuture = executorService.submit( + () -> performHealthCheck("MySQL", mysqlStatus, this::checkMySQLHealthSync)); + redisFuture = executorService.submit( + () -> performHealthCheck("Redis", redisStatus, this::checkRedisHealthSync)); + + awaitHealthChecks(mysqlFuture, redisFuture); + } catch (TimeoutException e) { + logger.warn("Health check aggregate timeout after {} seconds", getMaxTimeout()); + cancelFutures(mysqlFuture, redisFuture); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("Health check was interrupted"); + cancelFutures(mysqlFuture, redisFuture); + } catch (Exception e) { + logger.warn("Health check execution error: {}", e.getMessage()); + } + } + + private void awaitHealthChecks(Future mysqlFuture, Future redisFuture) throws TimeoutException, InterruptedException, ExecutionException { + long maxTimeout = getMaxTimeout(); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(maxTimeout); + + mysqlFuture.get(maxTimeout, TimeUnit.SECONDS); + long remainingNs = deadlineNs - System.nanoTime(); + + if (remainingNs > 0) { + redisFuture.get(remainingNs, TimeUnit.NANOSECONDS); + } else { + redisFuture.cancel(true); + } + } + + private long getMaxTimeout() { + return Math.max(MYSQL_TIMEOUT_SECONDS, REDIS_TIMEOUT_SECONDS) + 1; + } + + private void cancelFutures(Future mysqlFuture, Future redisFuture) { + if (mysqlFuture != null) mysqlFuture.cancel(true); + if (redisFuture != null) redisFuture.cancel(true); + } + private void ensurePopulated(Map status, String componentName) { if (!status.containsKey(STATUS_KEY)) { status.put(STATUS_KEY, STATUS_DOWN); @@ -199,8 +210,8 @@ private HealthCheckResult checkMySQLHealthSync() { try (ResultSet rs = stmt.executeQuery()) { if (rs.next()) { - // Basic health check passed, now run advanced checks with throttling - boolean isDegraded = performAdvancedMySQLChecksWithThrottle(connection); + // Basic health check passed, now run advanced checks asynchronously with fresh connection + boolean isDegraded = performAdvancedMySQLChecksWithThrottle(); return new HealthCheckResult(true, null, isDegraded); } } @@ -332,8 +343,8 @@ private String computeOverallStatus(Map> components) } // Internal advanced health checks for MySQL - do not expose details in responses - private boolean performAdvancedMySQLChecksWithThrottle(Connection connection) { - if (!advancedHealthChecksEnabled) { + private boolean performAdvancedMySQLChecksWithThrottle() { + if (!ADVANCED_HEALTH_CHECKS_ENABLED) { return false; // Advanced checks disabled } @@ -359,13 +370,27 @@ private boolean performAdvancedMySQLChecksWithThrottle(Connection connection) { (currentTime - lastAdvancedCheckTime) < ADVANCED_CHECKS_THROTTLE_SECONDS * 1000) { return cachedAdvancedCheckResult.isDegraded; } - - AdvancedCheckResult result = performAdvancedMySQLChecks(connection); - - // Cache the result + } finally { + advancedCheckLock.writeLock().unlock(); + } + + // Perform DB I/O outside the write lock to avoid lock contention + AdvancedCheckResult result; + try (Connection conn = dataSource.getConnection()) { + result = performAdvancedMySQLChecks(conn); + } catch (Exception ex) { + if (ex.getCause() instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + logger.debug("Could not acquire connection for advanced checks: {}", ex.getMessage()); + result = new AdvancedCheckResult(false); // don't mark degraded on acquisition failure + } + + // Re-acquire write lock only to update the cache atomically + advancedCheckLock.writeLock().lock(); + try { lastAdvancedCheckTime = currentTime; cachedAdvancedCheckResult = result; - return result.isDegraded; } finally { advancedCheckLock.writeLock().unlock(); @@ -381,11 +406,6 @@ private AdvancedCheckResult performAdvancedMySQLChecks(Connection connection) { hasIssues = true; } - if (hasDeadlocks(connection)) { - logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_DEADLOCK); - hasIssues = true; - } - if (hasSlowQueries(connection)) { logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_SLOW_QUERIES); hasIssues = true; @@ -409,7 +429,7 @@ private boolean hasLockWaits(Connection connection) { "WHERE (state = 'Waiting for table metadata lock' " + " OR state = 'Waiting for row lock' " + " OR state = 'Waiting for lock') " + - "AND user = USER()")) { + "AND user = SUBSTRING_INDEX(USER(), '@', 1)")) { stmt.setQueryTimeout(2); try (ResultSet rs = stmt.executeQuery()) { if (rs.next()) { @@ -423,41 +443,11 @@ private boolean hasLockWaits(Connection connection) { return false; } - private boolean hasDeadlocks(Connection connection) { - // Skip deadlock check if already disabled due to permissions - if (deadlockCheckDisabled) { - return false; - } - - try (PreparedStatement stmt = connection.prepareStatement("SHOW ENGINE INNODB STATUS")) { - stmt.setQueryTimeout(2); - try (ResultSet rs = stmt.executeQuery()) { - if (rs.next()) { - String innodbStatus = rs.getString(3); - return innodbStatus != null && innodbStatus.contains("LATEST DETECTED DEADLOCK"); - } - } - } catch (java.sql.SQLException e) { - // Check if this is a permission error - if (e.getMessage() != null && - (e.getMessage().contains("Access denied") || - e.getMessage().contains("permission"))) { - // Disable this check permanently after first permission error - deadlockCheckDisabled = true; - logger.warn("Deadlock check disabled: Insufficient privileges"); - } else { - logger.debug("Could not check for deadlocks"); - } - } catch (Exception e) { - logger.debug("Could not check for deadlocks"); - } - return false; - } - private boolean hasSlowQueries(Connection connection) { try (PreparedStatement stmt = connection.prepareStatement( "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + - "WHERE command != 'Sleep' AND time > ? AND user NOT IN ('event_scheduler', 'system user')")) { + "WHERE command != 'Sleep' AND time > ? " + + "AND user = SUBSTRING_INDEX(USER(), '@', 1)")) { stmt.setQueryTimeout(2); stmt.setInt(1, 10); // Queries running longer than 10 seconds try (ResultSet rs = stmt.executeQuery()) {