From 4687b3faee30f7ecc7d2cefd472e08ce6b64d973 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Tue, 17 Feb 2026 21:05:10 +0530 Subject: [PATCH 01/20] feat(health,version): add health and version endpoints --- pom.xml | 34 +- .../controller/health/HealthController.java | 118 +++++ .../controller/version/VersionController.java | 65 +-- .../service/health/HealthService.java | 419 ++++++++++++++++++ .../utils/JwtUserIdValidationFilter.java | 7 + 5 files changed, 598 insertions(+), 45 deletions(-) create mode 100644 src/main/java/com/iemr/common/identity/controller/health/HealthController.java create mode 100644 src/main/java/com/iemr/common/identity/service/health/HealthService.java diff --git a/pom.xml b/pom.xml index 5170d70..d6d41b7 100644 --- a/pom.xml +++ b/pom.xml @@ -245,6 +245,12 @@ h2 runtime + + + org.elasticsearch.client + elasticsearch-rest-client + 8.10.0 + @@ -286,31 +292,29 @@ 3.3.0 - pl.project13.maven - git-commit-id-plugin - 4.9.10 + io.github.git-commit-id + git-commit-id-maven-plugin + 7.0.0 get-the-git-infos revision + initialize - ${project.basedir}/.git - git - false true - - ${project.build.outputDirectory}/git.properties - - json - - false - false - -dirty - + ${project.build.outputDirectory}/git.properties + + ^git.branch$ + ^git.commit.id.abbrev$ + ^git.build.version$ + ^git.build.time$ + + false + false diff --git a/src/main/java/com/iemr/common/identity/controller/health/HealthController.java b/src/main/java/com/iemr/common/identity/controller/health/HealthController.java new file mode 100644 index 0000000..3091f0f --- /dev/null +++ b/src/main/java/com/iemr/common/identity/controller/health/HealthController.java @@ -0,0 +1,118 @@ +package com.iemr.common.identity.controller.health; + +import java.time.Instant; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import jakarta.servlet.http.Cookie; +import jakarta.servlet.http.HttpServletRequest; +import com.iemr.common.identity.service.health.HealthService; +import com.iemr.common.identity.utils.JwtAuthenticationUtil; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; + + +@RestController +@RequestMapping("/health") +@Tag(name = "Health Check", description = "APIs for checking infrastructure health status") +public class HealthController { + + private static final Logger logger = LoggerFactory.getLogger(HealthController.class); + + private final HealthService healthService; + private final JwtAuthenticationUtil jwtAuthenticationUtil; + + public HealthController(HealthService healthService, JwtAuthenticationUtil jwtAuthenticationUtil) { + this.healthService = healthService; + this.jwtAuthenticationUtil = jwtAuthenticationUtil; + } + @GetMapping + @Operation(summary = "Check infrastructure health", + description = "Returns the health status of MySQL, Redis, Elasticsearch, and other configured services") + @ApiResponses({ + @ApiResponse(responseCode = "200", description = "All checked components are UP"), + @ApiResponse(responseCode = "503", description = "One or more critical services are DOWN") + }) + public ResponseEntity> checkHealth(HttpServletRequest request) { + logger.info("Health check endpoint called"); + + try { + // Check if user is authenticated by verifying Authorization header + boolean isAuthenticated = isUserAuthenticated(request); + Map healthStatus = healthService.checkHealth(isAuthenticated); + String overallStatus = (String) healthStatus.get("status"); + + // Return 200 if overall status is UP, 503 if DOWN + HttpStatus httpStatus = "UP".equals(overallStatus) ? HttpStatus.OK : HttpStatus.SERVICE_UNAVAILABLE; + + logger.debug("Health check completed with status: {}", overallStatus); + return new ResponseEntity<>(healthStatus, httpStatus); + + } catch (Exception e) { + logger.error("Unexpected error during health check", e); + + // Return sanitized error response + Map errorResponse = Map.of( + "status", "DOWN", + "error", "Health check service unavailable", + "timestamp", Instant.now().toString() + ); + + return new ResponseEntity<>(errorResponse, HttpStatus.SERVICE_UNAVAILABLE); + } + } + + private boolean isUserAuthenticated(HttpServletRequest request) { + String token = null; + + // First, try to get token from JwtToken header + token = request.getHeader("JwtToken"); + + // If not found, try Authorization header + if (token == null || token.trim().isEmpty()) { + String authHeader = request.getHeader("Authorization"); + if (authHeader != null && !authHeader.trim().isEmpty()) { + // Extract token from "Bearer " format + token = authHeader.startsWith("Bearer ") + ? authHeader.substring(7) + : authHeader; + } + } + + // If still not found, try to get from cookies + if (token == null || token.trim().isEmpty()) { + token = getJwtTokenFromCookies(request); + } + + // Validate the token if found + if (token != null && !token.trim().isEmpty()) { + try { + return jwtAuthenticationUtil.validateUserIdAndJwtToken(token); + } catch (Exception e) { + logger.debug("JWT token validation failed: {}", e.getMessage()); + return false; + } + } + + return false; + } + + private String getJwtTokenFromCookies(HttpServletRequest request) { + Cookie[] cookies = request.getCookies(); + if (cookies != null) { + for (Cookie cookie : cookies) { + if (cookie.getName().equalsIgnoreCase("Jwttoken")) { + return cookie.getValue(); + } + } + } + return null; + } +} diff --git a/src/main/java/com/iemr/common/identity/controller/version/VersionController.java b/src/main/java/com/iemr/common/identity/controller/version/VersionController.java index 4435c3d..39eee9f 100644 --- a/src/main/java/com/iemr/common/identity/controller/version/VersionController.java +++ b/src/main/java/com/iemr/common/identity/controller/version/VersionController.java @@ -21,54 +21,59 @@ */ package com.iemr.common.identity.controller.version; -import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; -import com.iemr.common.identity.utils.response.OutputResponse; - import io.swagger.v3.oas.annotations.Operation; @RestController public class VersionController { - private Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + private final Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + private static final String UNKNOWN_VALUE = "unknown"; + @Operation(summary = "Get version information") - @GetMapping(value = "/version",consumes = "application/json", produces = "application/json") - public String versionInformation() { - OutputResponse output = new OutputResponse(); + @GetMapping(value = "/version", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity> versionInformation() { + Map response = new LinkedHashMap<>(); try { logger.info("version Controller Start"); - output.setResponse(readGitProperties()); - } catch (Exception e) { - output.setError(e); - } - + Properties gitProperties = loadGitProperties(); + response.put("buildTimestamp", gitProperties.getProperty("git.build.time", UNKNOWN_VALUE)); + response.put("version", gitProperties.getProperty("git.build.version", UNKNOWN_VALUE)); + response.put("branch", gitProperties.getProperty("git.branch", UNKNOWN_VALUE)); + response.put("commitHash", gitProperties.getProperty("git.commit.id.abbrev", UNKNOWN_VALUE)); + } catch (Exception e) { + logger.error("Failed to load version information", e); + response.put("buildTimestamp", UNKNOWN_VALUE); + response.put("version", UNKNOWN_VALUE); + response.put("branch", UNKNOWN_VALUE); + response.put("commitHash", UNKNOWN_VALUE); + } logger.info("version Controller End"); - return output.toString(); + return ResponseEntity.ok(response); } - private String readGitProperties() throws Exception { - ClassLoader classLoader = getClass().getClassLoader(); - InputStream inputStream = classLoader.getResourceAsStream("git.properties"); - - return readFromInputStream(inputStream); - } - private String readFromInputStream(InputStream inputStream) - throws IOException { - StringBuilder resultStringBuilder = new StringBuilder(); - try (BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) { - String line; - while ((line = br.readLine()) != null) { - resultStringBuilder.append(line).append("\n"); - } - } - return resultStringBuilder.toString(); + + private Properties loadGitProperties() throws IOException { + Properties properties = new Properties(); + try (InputStream input = getClass().getClassLoader() + .getResourceAsStream("git.properties")) { + if (input != null) { + properties.load(input); + } + } + return properties; } } diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java new file mode 100644 index 0000000..00bd086 --- /dev/null +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -0,0 +1,419 @@ +package com.iemr.common.identity.service.health; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Supplier; +import javax.sql.DataSource; +import jakarta.annotation.PostConstruct; +import org.apache.http.HttpHost; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RestClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.redis.core.RedisCallback; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.stereotype.Service; + +@Service +public class HealthService { + + private static final Logger logger = LoggerFactory.getLogger(HealthService.class); + private static final String STATUS_KEY = "status"; + private static final String DB_HEALTH_CHECK_QUERY = "SELECT 1 as health_check"; + private static final String DB_VERSION_QUERY = "SELECT VERSION()"; + private static final String STATUS_UP = "UP"; + private static final String STATUS_DOWN = "DOWN"; + private static final String UNKNOWN_VALUE = "unknown"; + private static final int REDIS_TIMEOUT_SECONDS = 3; + private static final ExecutorService executorService = Executors.newFixedThreadPool(4); + + private final DataSource dataSource; + private final RedisTemplate redisTemplate; + private final String dbUrl; + private final String redisHost; + private final int redisPort; + private final String elasticsearchHost; + private final int elasticsearchPort; + private final boolean elasticsearchEnabled; + private RestClient elasticsearchRestClient; + private boolean elasticsearchClientReady = false; + + public HealthService(DataSource dataSource, + @Autowired(required = false) RedisTemplate redisTemplate, + @Value("${spring.datasource.url:unknown}") String dbUrl, + @Value("${spring.data.redis.host:localhost}") String redisHost, + @Value("${spring.data.redis.port:6379}") int redisPort, + @Value("${elasticsearch.host:localhost}") String elasticsearchHost, + @Value("${elasticsearch.port:9200}") int elasticsearchPort, + @Value("${elasticsearch.enabled:false}") boolean elasticsearchEnabled) { + this.dataSource = dataSource; + this.redisTemplate = redisTemplate; + this.dbUrl = dbUrl; + this.redisHost = redisHost; + this.redisPort = redisPort; + this.elasticsearchHost = elasticsearchHost; + this.elasticsearchPort = elasticsearchPort; + this.elasticsearchEnabled = elasticsearchEnabled; + } + + @PostConstruct + public void init() { + // Initialize Elasticsearch RestClient if enabled + if (elasticsearchEnabled) { + initializeElasticsearchClient(); + } + } + + private void initializeElasticsearchClient() { + try { + this.elasticsearchRestClient = RestClient.builder( + new HttpHost(elasticsearchHost, elasticsearchPort, "http") + ) + .setRequestConfigCallback(requestConfigBuilder -> + requestConfigBuilder + .setConnectTimeout(3000) + .setSocketTimeout(3000) + ) + .build(); + this.elasticsearchClientReady = true; + logger.info("Elasticsearch RestClient initialized for {}:{}", elasticsearchHost, elasticsearchPort); + } catch (Exception e) { + logger.warn("Failed to initialize Elasticsearch RestClient: {}", e.getMessage()); + this.elasticsearchClientReady = false; + } + } + + public Map checkHealth(boolean includeDetails) { + Map healthStatus = new LinkedHashMap<>(); + Map components = new LinkedHashMap<>(); + boolean overallHealth = true; + + Map mysqlStatus = checkMySQLHealth(includeDetails); + components.put("mysql", mysqlStatus); + if (!isHealthy(mysqlStatus)) { + overallHealth = false; + } + + if (redisTemplate != null) { + Map redisStatus = checkRedisHealth(includeDetails); + components.put("redis", redisStatus); + if (!isHealthy(redisStatus)) { + overallHealth = false; + } + } + + if (elasticsearchEnabled && elasticsearchClientReady) { + Map elasticsearchStatus = checkElasticsearchHealth(includeDetails); + components.put("elasticsearch", elasticsearchStatus); + if (!isHealthy(elasticsearchStatus)) { + overallHealth = false; + } + } + + healthStatus.put(STATUS_KEY, overallHealth ? STATUS_UP : STATUS_DOWN); + healthStatus.put("timestamp", Instant.now().toString()); + healthStatus.put("components", components); + logger.info("Health check completed - Overall status: {}", overallHealth ? STATUS_UP : STATUS_DOWN); + + return healthStatus; + } + + public Map checkHealth() { + return checkHealth(true); + } + + private Map checkMySQLHealth(boolean includeDetails) { + Map details = new LinkedHashMap<>(); + details.put("type", "MySQL"); + + if (includeDetails) { + details.put("host", extractHost(dbUrl)); + details.put("port", extractPort(dbUrl)); + details.put("database", extractDatabaseName(dbUrl)); + } + + return performHealthCheck("MySQL", details, () -> { + try { + try (Connection connection = dataSource.getConnection()) { + if (connection.isValid(2)) { + try (PreparedStatement stmt = connection.prepareStatement(DB_HEALTH_CHECK_QUERY)) { + stmt.setQueryTimeout(3); + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next() && rs.getInt(1) == 1) { + String version = includeDetails ? getMySQLVersion(connection) : null; + return new HealthCheckResult(true, version, null); + } + } + } + } + return new HealthCheckResult(false, null, "Connection validation failed"); + } + } catch (Exception e) { + logger.error("MySQL health check exception: {}", e.getMessage(), e); + throw new IllegalStateException("MySQL connection failed: " + e.getMessage(), e); + } + }); + } + + private Map checkRedisHealth(boolean includeDetails) { + Map details = new LinkedHashMap<>(); + details.put("type", "Redis"); + + if (includeDetails) { + details.put("host", redisHost); + details.put("port", redisPort); + } + + return performHealthCheck("Redis", details, () -> { + try { + // Wrap PING in CompletableFuture with timeout + String pong = CompletableFuture.supplyAsync(() -> + redisTemplate.execute((RedisCallback) connection -> connection.ping()), + executorService + ).get(REDIS_TIMEOUT_SECONDS, TimeUnit.SECONDS); + + if ("PONG".equals(pong)) { + String version = includeDetails ? getRedisVersionWithTimeout() : null; + return new HealthCheckResult(true, version, null); + } + return new HealthCheckResult(false, null, "Ping returned unexpected response"); + } catch (TimeoutException e) { + return new HealthCheckResult(false, null, "Redis ping timed out after " + REDIS_TIMEOUT_SECONDS + " seconds"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return new HealthCheckResult(false, null, "Redis health check was interrupted"); + } catch (Exception e) { + throw new IllegalStateException("Redis health check failed", e); + } + }); + } + + private Map checkElasticsearchHealth(boolean includeDetails) { + Map details = new LinkedHashMap<>(); + details.put("type", "Elasticsearch"); + + if (includeDetails) { + details.put("host", elasticsearchHost); + details.put("port", elasticsearchPort); + } + + return performHealthCheck("Elasticsearch", details, () -> { + if (!elasticsearchClientReady || elasticsearchRestClient == null) { + logger.debug("Elasticsearch RestClient not ready"); + return new HealthCheckResult(false, null, "Elasticsearch client not ready"); + } + + try { + // Execute a simple cluster health request + Request request = new Request("GET", "/_cluster/health"); + var response = elasticsearchRestClient.performRequest(request); + + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == 200) { + logger.debug("Elasticsearch health check successful"); + return new HealthCheckResult(true, "Elasticsearch 8.10.0", null); + } + return new HealthCheckResult(false, null, "HTTP " + statusCode); + } catch (java.net.ConnectException e) { + logger.debug("Elasticsearch connection refused on {}:{}", elasticsearchHost, elasticsearchPort); + return new HealthCheckResult(false, null, "Connection refused"); + } catch (java.io.IOException e) { + logger.debug("Elasticsearch IO error: {}", e.getMessage()); + return new HealthCheckResult(false, null, "IO Error: " + e.getMessage()); + } catch (Exception e) { + logger.debug("Elasticsearch error: {} - {}", e.getClass().getSimpleName(), e.getMessage()); + return new HealthCheckResult(false, null, e.getMessage()); + } + }); + } + + private Map performHealthCheck(String componentName, + Map details, + Supplier checker) { + Map status = new LinkedHashMap<>(); + long startTime = System.currentTimeMillis(); + + try { + HealthCheckResult result = checker.get(); + long responseTime = System.currentTimeMillis() - startTime; + + details.put("responseTimeMs", responseTime); + + if (result.isHealthy) { + logger.debug("{} health check: UP ({}ms)", componentName, responseTime); + status.put(STATUS_KEY, STATUS_UP); + if (result.version != null) { + details.put("version", result.version); + } + } else { + String safeError = result.error != null ? result.error : "Health check failed"; + logger.warn("{} health check failed: {}", componentName, safeError); + status.put(STATUS_KEY, STATUS_DOWN); + details.put("error", safeError); + details.put("errorType", "CheckFailed"); + } + + status.put("details", details); + return status; + + } catch (Exception e) { + long responseTime = System.currentTimeMillis() - startTime; + + logger.error("{} health check failed with exception: {}", componentName, e.getMessage(), e); + + String errorMessage = e.getCause() != null + ? e.getCause().getMessage() + : e.getMessage(); + + status.put(STATUS_KEY, STATUS_DOWN); + details.put("responseTimeMs", responseTime); + details.put("error", errorMessage != null ? errorMessage : "Health check failed"); + details.put("errorType", "InternalError"); + status.put("details", details); + + return status; + } + } + + private boolean isHealthy(Map componentStatus) { + return STATUS_UP.equals(componentStatus.get(STATUS_KEY)); + } + + private String getMySQLVersion(Connection connection) { + try (PreparedStatement stmt = connection.prepareStatement(DB_VERSION_QUERY); + ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + return rs.getString(1); + } + } catch (Exception e) { + logger.debug("Could not retrieve MySQL version", e); + } + return null; + } + + private String getRedisVersion() { + try { + Properties info = redisTemplate.execute((RedisCallback) connection -> + connection.serverCommands().info("server") + ); + if (info != null && info.containsKey("redis_version")) { + return info.getProperty("redis_version"); + } + } catch (Exception e) { + logger.debug("Could not retrieve Redis version", e); + } + return null; + } + + private String getRedisVersionWithTimeout() { + try { + return CompletableFuture.supplyAsync(this::getRedisVersion, executorService) + .get(REDIS_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } catch (TimeoutException e) { + logger.debug("Redis version retrieval timed out"); + return null; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.debug("Redis version retrieval was interrupted"); + return null; + } catch (Exception e) { + logger.debug("Could not retrieve Redis version with timeout", e); + return null; + } + } + + private String getElasticsearchVersion(RestClient restClient) { + try { + Request request = new Request("GET", "/"); + var response = restClient.performRequest(request); + + if (response.getStatusLine().getStatusCode() == 200) { + // The response typically contains JSON with version info + return "Elasticsearch"; // Simplified extraction + } + } catch (Exception e) { + logger.debug("Could not retrieve Elasticsearch version", e); + } + return null; + } + + private String extractHost(String jdbcUrl) { + if (jdbcUrl == null || UNKNOWN_VALUE.equals(jdbcUrl)) { + return UNKNOWN_VALUE; + } + try { + String withoutPrefix = jdbcUrl.replaceFirst("jdbc:mysql://", ""); + int slashIndex = withoutPrefix.indexOf('/'); + String hostPort = slashIndex > 0 + ? withoutPrefix.substring(0, slashIndex) + : withoutPrefix; + int colonIndex = hostPort.indexOf(':'); + return colonIndex > 0 ? hostPort.substring(0, colonIndex) : hostPort; + } catch (Exception e) { + logger.debug("Could not extract host from URL", e); + } + return UNKNOWN_VALUE; + } + + private String extractPort(String jdbcUrl) { + if (jdbcUrl == null || UNKNOWN_VALUE.equals(jdbcUrl)) { + return UNKNOWN_VALUE; + } + try { + String withoutPrefix = jdbcUrl.replaceFirst("jdbc:mysql://", ""); + int slashIndex = withoutPrefix.indexOf('/'); + String hostPort = slashIndex > 0 + ? withoutPrefix.substring(0, slashIndex) + : withoutPrefix; + int colonIndex = hostPort.indexOf(':'); + return colonIndex > 0 ? hostPort.substring(colonIndex + 1) : "3306"; + } catch (Exception e) { + logger.debug("Could not extract port from URL", e); + } + return "3306"; + } + + private String extractDatabaseName(String jdbcUrl) { + if (jdbcUrl == null || UNKNOWN_VALUE.equals(jdbcUrl)) { + return UNKNOWN_VALUE; + } + try { + int lastSlash = jdbcUrl.lastIndexOf('/'); + if (lastSlash >= 0 && lastSlash < jdbcUrl.length() - 1) { + String afterSlash = jdbcUrl.substring(lastSlash + 1); + int queryStart = afterSlash.indexOf('?'); + if (queryStart > 0) { + return afterSlash.substring(0, queryStart); + } + return afterSlash; + } + } catch (Exception e) { + logger.debug("Could not extract database name from URL", e); + } + return UNKNOWN_VALUE; + } + + private static class HealthCheckResult { + final boolean isHealthy; + final String version; + final String error; + + HealthCheckResult(boolean isHealthy, String version, String error) { + this.isHealthy = isHealthy; + this.version = version; + this.error = error; + } + } +} diff --git a/src/main/java/com/iemr/common/identity/utils/JwtUserIdValidationFilter.java b/src/main/java/com/iemr/common/identity/utils/JwtUserIdValidationFilter.java index 6d5c55f..d383cf5 100644 --- a/src/main/java/com/iemr/common/identity/utils/JwtUserIdValidationFilter.java +++ b/src/main/java/com/iemr/common/identity/utils/JwtUserIdValidationFilter.java @@ -45,6 +45,13 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo String path = request.getRequestURI(); logger.info("JwtUserIdValidationFilter invoked for path: " + path); + // Skip JWT validation for public endpoints + if (path.endsWith("/health") || path.endsWith("/version")) { + logger.info("Public endpoint accessed: " + path + " - skipping JWT validation"); + filterChain.doFilter(servletRequest, servletResponse); + return; + } + // Log cookies for debugging Cookie[] cookies = request.getCookies(); if (cookies != null) { From 1c2468b786c1bfc0540aae61db3fccfdf567d7cf Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Tue, 17 Feb 2026 21:06:49 +0530 Subject: [PATCH 02/20] feat(health,version): add health and version endpoints without auth --- .../controller/health/HealthController.java | 22 +++++++++++++++++++ .../service/health/HealthService.java | 22 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/src/main/java/com/iemr/common/identity/controller/health/HealthController.java b/src/main/java/com/iemr/common/identity/controller/health/HealthController.java index 3091f0f..89de260 100644 --- a/src/main/java/com/iemr/common/identity/controller/health/HealthController.java +++ b/src/main/java/com/iemr/common/identity/controller/health/HealthController.java @@ -1,3 +1,25 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + package com.iemr.common.identity.controller.health; import java.time.Instant; diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java index 00bd086..59100e9 100644 --- a/src/main/java/com/iemr/common/identity/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -1,3 +1,25 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + package com.iemr.common.identity.service.health; import java.sql.Connection; From ffc037018288b84b65e09623c3a6682207f273a1 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Tue, 17 Feb 2026 21:20:25 +0530 Subject: [PATCH 03/20] fix(health): remove unused private methods --- .../service/health/HealthService.java | 28 ++++++------------- .../utils/JwtUserIdValidationFilter.java | 8 +++--- 2 files changed, 12 insertions(+), 24 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java index 59100e9..bb98f48 100644 --- a/src/main/java/com/iemr/common/identity/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -58,6 +58,7 @@ public class HealthService { private static final String STATUS_UP = "UP"; private static final String STATUS_DOWN = "DOWN"; private static final String UNKNOWN_VALUE = "unknown"; + private static final String ELASTICSEARCH_TYPE = "Elasticsearch"; private static final int REDIS_TIMEOUT_SECONDS = 3; private static final ExecutorService executorService = Executors.newFixedThreadPool(4); @@ -224,14 +225,14 @@ private Map checkRedisHealth(boolean includeDetails) { private Map checkElasticsearchHealth(boolean includeDetails) { Map details = new LinkedHashMap<>(); - details.put("type", "Elasticsearch"); + details.put("type", ELASTICSEARCH_TYPE); if (includeDetails) { details.put("host", elasticsearchHost); details.put("port", elasticsearchPort); } - return performHealthCheck("Elasticsearch", details, () -> { + return performHealthCheck(ELASTICSEARCH_TYPE, details, () -> { if (!elasticsearchClientReady || elasticsearchRestClient == null) { logger.debug("Elasticsearch RestClient not ready"); return new HealthCheckResult(false, null, "Elasticsearch client not ready"); @@ -244,18 +245,18 @@ private Map checkElasticsearchHealth(boolean includeDetails) { int statusCode = response.getStatusLine().getStatusCode(); if (statusCode == 200) { - logger.debug("Elasticsearch health check successful"); + logger.debug("{} health check successful", ELASTICSEARCH_TYPE); return new HealthCheckResult(true, "Elasticsearch 8.10.0", null); } return new HealthCheckResult(false, null, "HTTP " + statusCode); } catch (java.net.ConnectException e) { - logger.debug("Elasticsearch connection refused on {}:{}", elasticsearchHost, elasticsearchPort); + logger.error("{} connection refused on {}:{}", ELASTICSEARCH_TYPE, elasticsearchHost, elasticsearchPort, e); return new HealthCheckResult(false, null, "Connection refused"); } catch (java.io.IOException e) { - logger.debug("Elasticsearch IO error: {}", e.getMessage()); + logger.error("{} IO error: {}", ELASTICSEARCH_TYPE, e.getMessage(), e); return new HealthCheckResult(false, null, "IO Error: " + e.getMessage()); } catch (Exception e) { - logger.debug("Elasticsearch error: {} - {}", e.getClass().getSimpleName(), e.getMessage()); + logger.error("{} error: {} - {}", ELASTICSEARCH_TYPE, e.getClass().getSimpleName(), e.getMessage(), e); return new HealthCheckResult(false, null, e.getMessage()); } }); @@ -356,20 +357,7 @@ private String getRedisVersionWithTimeout() { } } - private String getElasticsearchVersion(RestClient restClient) { - try { - Request request = new Request("GET", "/"); - var response = restClient.performRequest(request); - - if (response.getStatusLine().getStatusCode() == 200) { - // The response typically contains JSON with version info - return "Elasticsearch"; // Simplified extraction - } - } catch (Exception e) { - logger.debug("Could not retrieve Elasticsearch version", e); - } - return null; - } + private String extractHost(String jdbcUrl) { if (jdbcUrl == null || UNKNOWN_VALUE.equals(jdbcUrl)) { diff --git a/src/main/java/com/iemr/common/identity/utils/JwtUserIdValidationFilter.java b/src/main/java/com/iemr/common/identity/utils/JwtUserIdValidationFilter.java index d383cf5..2b2e7b5 100644 --- a/src/main/java/com/iemr/common/identity/utils/JwtUserIdValidationFilter.java +++ b/src/main/java/com/iemr/common/identity/utils/JwtUserIdValidationFilter.java @@ -43,11 +43,11 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo return; } String path = request.getRequestURI(); - logger.info("JwtUserIdValidationFilter invoked for path: " + path); + logger.info("JwtUserIdValidationFilter invoked for path: {}", path); // Skip JWT validation for public endpoints if (path.endsWith("/health") || path.endsWith("/version")) { - logger.info("Public endpoint accessed: " + path + " - skipping JWT validation"); + logger.info("Public endpoint accessed: {} - skipping JWT validation", path); filterChain.doFilter(servletRequest, servletResponse); return; } @@ -83,10 +83,10 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo } } else { String userAgent = request.getHeader("User-Agent"); - logger.info("User-Agent: " + userAgent); + logger.info("User-Agent: {}", userAgent); if (userAgent != null && isMobileClient(userAgent) && authHeader != null) { try { - logger.info("Common-API incoming userAget : " + userAgent); + logger.info("Common-API incoming userAget: {}", userAgent); UserAgentContext.setUserAgent(userAgent); filterChain.doFilter(servletRequest, servletResponse); } finally { From 853cfa3c1b85fdad179c7a80ba171b9c1197ff8d Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Tue, 17 Feb 2026 21:24:37 +0530 Subject: [PATCH 04/20] fix(health): fix exception issue --- .../com/iemr/common/identity/service/health/HealthService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java index bb98f48..680079c 100644 --- a/src/main/java/com/iemr/common/identity/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -184,7 +184,6 @@ private Map checkMySQLHealth(boolean includeDetails) { return new HealthCheckResult(false, null, "Connection validation failed"); } } catch (Exception e) { - logger.error("MySQL health check exception: {}", e.getMessage(), e); throw new IllegalStateException("MySQL connection failed: " + e.getMessage(), e); } }); From 6422265b40893bfe20568a29ced97ae555ed1ed7 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Tue, 17 Feb 2026 21:36:01 +0530 Subject: [PATCH 05/20] fix(health): redact error details for unauthenticated health checks --- pom.xml | 2 +- .../service/health/HealthService.java | 33 ++++++++++++++----- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index d6d41b7..3755281 100644 --- a/pom.xml +++ b/pom.xml @@ -294,7 +294,7 @@ io.github.git-commit-id git-commit-id-maven-plugin - 7.0.0 + 9.0.2 get-the-git-infos diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java index 680079c..af9067a 100644 --- a/src/main/java/com/iemr/common/identity/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -60,9 +60,9 @@ public class HealthService { private static final String UNKNOWN_VALUE = "unknown"; private static final String ELASTICSEARCH_TYPE = "Elasticsearch"; private static final int REDIS_TIMEOUT_SECONDS = 3; - private static final ExecutorService executorService = Executors.newFixedThreadPool(4); private final DataSource dataSource; + private final ExecutorService executorService = Executors.newFixedThreadPool(4); private final RedisTemplate redisTemplate; private final String dbUrl; private final String redisHost; @@ -99,6 +99,18 @@ public void init() { } } + @jakarta.annotation.PreDestroy + public void cleanup() { + executorService.shutdownNow(); + if (elasticsearchRestClient != null) { + try { + elasticsearchRestClient.close(); + } catch (IOException e) { + logger.warn("Error closing Elasticsearch client", e); + } + } + } + private void initializeElasticsearchClient() { try { this.elasticsearchRestClient = RestClient.builder( @@ -186,7 +198,7 @@ private Map checkMySQLHealth(boolean includeDetails) { } catch (Exception e) { throw new IllegalStateException("MySQL connection failed: " + e.getMessage(), e); } - }); + }, includeDetails); } private Map checkRedisHealth(boolean includeDetails) { @@ -219,7 +231,7 @@ private Map checkRedisHealth(boolean includeDetails) { } catch (Exception e) { throw new IllegalStateException("Redis health check failed", e); } - }); + }, includeDetails); } private Map checkElasticsearchHealth(boolean includeDetails) { @@ -245,7 +257,7 @@ private Map checkElasticsearchHealth(boolean includeDetails) { int statusCode = response.getStatusLine().getStatusCode(); if (statusCode == 200) { logger.debug("{} health check successful", ELASTICSEARCH_TYPE); - return new HealthCheckResult(true, "Elasticsearch 8.10.0", null); + return new HealthCheckResult(true, null, null); } return new HealthCheckResult(false, null, "HTTP " + statusCode); } catch (java.net.ConnectException e) { @@ -258,12 +270,13 @@ private Map checkElasticsearchHealth(boolean includeDetails) { logger.error("{} error: {} - {}", ELASTICSEARCH_TYPE, e.getClass().getSimpleName(), e.getMessage(), e); return new HealthCheckResult(false, null, e.getMessage()); } - }); + }, includeDetails); } private Map performHealthCheck(String componentName, Map details, - Supplier checker) { + Supplier checker, + boolean includeDetails) { Map status = new LinkedHashMap<>(); long startTime = System.currentTimeMillis(); @@ -283,7 +296,9 @@ private Map performHealthCheck(String componentName, String safeError = result.error != null ? result.error : "Health check failed"; logger.warn("{} health check failed: {}", componentName, safeError); status.put(STATUS_KEY, STATUS_DOWN); - details.put("error", safeError); + if (includeDetails) { + details.put("error", safeError); + } details.put("errorType", "CheckFailed"); } @@ -301,7 +316,9 @@ private Map performHealthCheck(String componentName, status.put(STATUS_KEY, STATUS_DOWN); details.put("responseTimeMs", responseTime); - details.put("error", errorMessage != null ? errorMessage : "Health check failed"); + if (includeDetails) { + details.put("error", errorMessage != null ? errorMessage : "Health check failed"); + } details.put("errorType", "InternalError"); status.put("details", details); From 00e34b739832507baef07eac7c0db2a0be8451e7 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Tue, 17 Feb 2026 21:39:28 +0530 Subject: [PATCH 06/20] fix code quality issues and reduce cognitive complexity --- .../service/health/HealthService.java | 68 +++++++++++-------- .../utils/JwtUserIdValidationFilter.java | 4 +- 2 files changed, 42 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java index af9067a..4af97df 100644 --- a/src/main/java/com/iemr/common/identity/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -287,19 +287,9 @@ private Map performHealthCheck(String componentName, details.put("responseTimeMs", responseTime); if (result.isHealthy) { - logger.debug("{} health check: UP ({}ms)", componentName, responseTime); - status.put(STATUS_KEY, STATUS_UP); - if (result.version != null) { - details.put("version", result.version); - } + buildHealthyStatus(status, details, componentName, responseTime, result); } else { - String safeError = result.error != null ? result.error : "Health check failed"; - logger.warn("{} health check failed: {}", componentName, safeError); - status.put(STATUS_KEY, STATUS_DOWN); - if (includeDetails) { - details.put("error", safeError); - } - details.put("errorType", "CheckFailed"); + buildUnhealthyStatus(status, details, componentName, result, includeDetails); } status.put("details", details); @@ -307,23 +297,45 @@ private Map performHealthCheck(String componentName, } catch (Exception e) { long responseTime = System.currentTimeMillis() - startTime; - - logger.error("{} health check failed with exception: {}", componentName, e.getMessage(), e); - - String errorMessage = e.getCause() != null - ? e.getCause().getMessage() - : e.getMessage(); - - status.put(STATUS_KEY, STATUS_DOWN); - details.put("responseTimeMs", responseTime); - if (includeDetails) { - details.put("error", errorMessage != null ? errorMessage : "Health check failed"); - } - details.put("errorType", "InternalError"); - status.put("details", details); - - return status; + return buildExceptionStatus(status, details, componentName, e, includeDetails, responseTime); + } + } + + private void buildHealthyStatus(Map status, Map details, + String componentName, long responseTime, HealthCheckResult result) { + logger.debug("{} health check: UP ({}ms)", componentName, responseTime); + status.put(STATUS_KEY, STATUS_UP); + if (result.version != null) { + details.put("version", result.version); + } + } + + private void buildUnhealthyStatus(Map status, Map details, + String componentName, HealthCheckResult result, boolean includeDetails) { + String safeError = result.error != null ? result.error : "Health check failed"; + logger.warn("{} health check failed: {}", componentName, safeError); + status.put(STATUS_KEY, STATUS_DOWN); + if (includeDetails) { + details.put("error", safeError); + } + details.put("errorType", "CheckFailed"); + } + + private Map buildExceptionStatus(Map status, Map details, + String componentName, Exception e, boolean includeDetails, long responseTime) { + logger.error("{} health check failed with exception: {}", componentName, e.getMessage(), e); + + String errorMessage = e.getCause() != null ? e.getCause().getMessage() : e.getMessage(); + + status.put(STATUS_KEY, STATUS_DOWN); + details.put("responseTimeMs", responseTime); + if (includeDetails) { + details.put("error", errorMessage != null ? errorMessage : "Health check failed"); } + details.put("errorType", "InternalError"); + status.put("details", details); + + return status; } private boolean isHealthy(Map componentStatus) { diff --git a/src/main/java/com/iemr/common/identity/utils/JwtUserIdValidationFilter.java b/src/main/java/com/iemr/common/identity/utils/JwtUserIdValidationFilter.java index 2b2e7b5..c63cf3c 100644 --- a/src/main/java/com/iemr/common/identity/utils/JwtUserIdValidationFilter.java +++ b/src/main/java/com/iemr/common/identity/utils/JwtUserIdValidationFilter.java @@ -46,7 +46,7 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo logger.info("JwtUserIdValidationFilter invoked for path: {}", path); // Skip JWT validation for public endpoints - if (path.endsWith("/health") || path.endsWith("/version")) { + if (path.equals("/health") || path.equals("/version")) { logger.info("Public endpoint accessed: {} - skipping JWT validation", path); filterChain.doFilter(servletRequest, servletResponse); return; @@ -86,7 +86,7 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo logger.info("User-Agent: {}", userAgent); if (userAgent != null && isMobileClient(userAgent) && authHeader != null) { try { - logger.info("Common-API incoming userAget: {}", userAgent); + logger.info("Common-API incoming userAgent: {}", userAgent); UserAgentContext.setUserAgent(userAgent); filterChain.doFilter(servletRequest, servletResponse); } finally { From 933e1cbe878860c7c4132b05c2478b68789b5e64 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Thu, 19 Feb 2026 09:46:37 +0530 Subject: [PATCH 07/20] feat(health): add MySQL health endpoint --- .../controller/health/HealthController.java | 5 +- .../service/health/HealthService.java | 300 ++++++++++++++++-- 2 files changed, 272 insertions(+), 33 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/controller/health/HealthController.java b/src/main/java/com/iemr/common/identity/controller/health/HealthController.java index 89de260..ef06de0 100644 --- a/src/main/java/com/iemr/common/identity/controller/health/HealthController.java +++ b/src/main/java/com/iemr/common/identity/controller/health/HealthController.java @@ -66,9 +66,8 @@ public ResponseEntity> checkHealth(HttpServletRequest reques logger.info("Health check endpoint called"); try { - // Check if user is authenticated by verifying Authorization header - boolean isAuthenticated = isUserAuthenticated(request); - Map healthStatus = healthService.checkHealth(isAuthenticated); + // Always include detailed metrics in health response + Map healthStatus = healthService.checkHealth(true); String overallStatus = (String) healthStatus.get("status"); // Return 200 if overall status is UP, 503 if DOWN diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java index 4af97df..5e38b87 100644 --- a/src/main/java/com/iemr/common/identity/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -22,6 +22,7 @@ package com.iemr.common.identity.service.health; +import java.io.IOException; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -172,12 +173,6 @@ public Map checkHealth() { private Map checkMySQLHealth(boolean includeDetails) { Map details = new LinkedHashMap<>(); details.put("type", "MySQL"); - - if (includeDetails) { - details.put("host", extractHost(dbUrl)); - details.put("port", extractPort(dbUrl)); - details.put("database", extractDatabaseName(dbUrl)); - } return performHealthCheck("MySQL", details, () -> { try { @@ -187,8 +182,7 @@ private Map checkMySQLHealth(boolean includeDetails) { stmt.setQueryTimeout(3); try (ResultSet rs = stmt.executeQuery()) { if (rs.next() && rs.getInt(1) == 1) { - String version = includeDetails ? getMySQLVersion(connection) : null; - return new HealthCheckResult(true, version, null); + return new HealthCheckResult(true, null, null); } } } @@ -200,15 +194,275 @@ private Map checkMySQLHealth(boolean includeDetails) { } }, includeDetails); } + + private void addAdvancedMySQLMetrics(Connection connection, Map details) { + try { + // Only add advanced metrics for MySQL, not for H2 or other databases + if (!isMySQLDatabase(connection)) { + logger.debug("Advanced metrics only supported for MySQL, skipping for this database"); + return; + } + + Map metrics = new LinkedHashMap<>(); + + // Get server status variables (uptime, connections, etc.) + Map statusVars = getMySQLStatusVariables(connection); + + if (!statusVars.isEmpty()) { + // Connections + Map connections = new LinkedHashMap<>(); + String threads = statusVars.get("Threads_connected"); + String maxConnections = statusVars.get("max_connections"); + if (threads != null && maxConnections != null) { + connections.put("active", Integer.parseInt(threads)); + connections.put("max", Integer.parseInt(maxConnections)); + try { + int active = Integer.parseInt(threads); + int max = Integer.parseInt(maxConnections); + connections.put("usage_percent", (active * 100) / max); + } catch (NumberFormatException e) { + logger.debug("Could not calculate connection usage percent"); + } + } + if (!connections.isEmpty()) { + metrics.put("connections", connections); + } + + // Uptime + String uptime = statusVars.get("Uptime"); + if (uptime != null) { + try { + long uptimeSeconds = Long.parseLong(uptime); + metrics.put("uptime_seconds", uptimeSeconds); + metrics.put("uptime_hours", uptimeSeconds / 3600); + } catch (NumberFormatException e) { + logger.debug("Could not parse uptime"); + } + } + + // Slow queries + String slowQueries = statusVars.get("Slow_queries"); + if (slowQueries != null) { + metrics.put("slow_queries", Integer.parseInt(slowQueries)); + } + + // Questions (total queries) + String questions = statusVars.get("Questions"); + if (questions != null) { + metrics.put("total_queries", Long.parseLong(questions)); + } + } + + // Database size + Map database = new LinkedHashMap<>(); + try (PreparedStatement stmt = connection.prepareStatement( + "SELECT table_schema, ROUND(SUM(data_length + index_length) / 1024 / 1024, 2) as size_mb " + + "FROM information_schema.tables WHERE table_schema = database() GROUP BY table_schema")) { + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + double sizeMb = rs.getDouble("size_mb"); + database.put("size_mb", sizeMb); + } + } + } catch (Exception e) { + logger.debug("Could not retrieve database size", e); + } + + // Table count + try (PreparedStatement stmt = connection.prepareStatement( + "SELECT COUNT(*) as table_count FROM information_schema.tables WHERE table_schema = database()")) { + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + database.put("tables_count", rs.getInt("table_count")); + } + } + } catch (Exception e) { + logger.debug("Could not retrieve table count", e); + } + + if (!database.isEmpty()) { + metrics.put("database", database); + } + + // Add deep health checks (locks, stuck processes, deadlocks) + addDeepMySQLHealthChecks(connection, metrics); + + if (!metrics.isEmpty()) { + details.put("metrics", metrics); + } + + } catch (Exception e) { + logger.debug("Error retrieving advanced MySQL metrics", e); + } + } + + private boolean isMySQLDatabase(Connection connection) { + try { + String databaseProductName = connection.getMetaData().getDatabaseProductName(); + return databaseProductName != null && databaseProductName.toLowerCase().contains("mysql"); + } catch (Exception e) { + logger.debug("Could not determine database type", e); + return false; + } + } + + private void addDeepMySQLHealthChecks(Connection connection, Map metrics) { + // Check for table locks + checkTableLocks(connection, metrics); + + // Check for stuck/long-running queries + checkStuckProcesses(connection, metrics); + + // Check for InnoDB deadlocks + checkInnoDBStatus(connection, metrics); + } + + private void checkTableLocks(Connection connection, Map metrics) { + try (PreparedStatement stmt = connection.prepareStatement( + "SELECT OBJECT_SCHEMA, OBJECT_NAME, COUNT(*) as lock_count " + + "FROM INFORMATION_SCHEMA.METADATA_LOCKS " + + "WHERE OBJECT_SCHEMA != 'mysql' AND OBJECT_SCHEMA != 'information_schema' " + + "GROUP BY OBJECT_SCHEMA, OBJECT_NAME")) { + try (ResultSet rs = stmt.executeQuery()) { + Map locks = new LinkedHashMap<>(); + int totalLocks = 0; + java.util.List lockedTables = new java.util.ArrayList<>(); + + while (rs.next()) { + totalLocks += rs.getInt("lock_count"); + String tableName = rs.getString("OBJECT_SCHEMA") + "." + rs.getString("OBJECT_NAME"); + lockedTables.add(tableName); + } + + if (totalLocks > 0) { + locks.put("is_locked", true); + locks.put("locked_tables_count", lockedTables.size()); + locks.put("locked_tables", lockedTables); + locks.put("total_locks", totalLocks); + locks.put("severity", totalLocks > 5 ? "CRITICAL" : "WARNING"); + metrics.put("table_locks", locks); + logger.warn("MySQL: {} table locks detected on {} tables", totalLocks, lockedTables.size()); + } else { + locks.put("is_locked", false); + locks.put("locked_tables_count", 0); + metrics.put("table_locks", locks); + } + } + } catch (Exception e) { + logger.debug("Could not check table locks (might not support METADATA_LOCKS)", e); + } + } + + private void checkStuckProcesses(Connection connection, Map metrics) { + try (PreparedStatement stmt = connection.prepareStatement( + "SELECT ID, USER, HOST, DB, TIME, COMMAND, STATE, INFO " + + "FROM INFORMATION_SCHEMA.PROCESSLIST " + + "WHERE COMMAND != 'Sleep' AND TIME > 300")) { // Processes running > 5 minutes + try (ResultSet rs = stmt.executeQuery()) { + Map processes = new LinkedHashMap<>(); + java.util.List> stuckQueries = new java.util.ArrayList<>(); + int totalRunning = 0; + long longestQuerySeconds = 0; + + while (rs.next()) { + long queryTime = rs.getLong("TIME"); + longestQuerySeconds = Math.max(longestQuerySeconds, queryTime); + + Map query = new LinkedHashMap<>(); + query.put("id", rs.getInt("ID")); + query.put("user", rs.getString("USER")); + query.put("command", rs.getString("COMMAND")); + query.put("time_seconds", queryTime); + query.put("state", rs.getString("STATE")); + + stuckQueries.add(query); + totalRunning++; + } + + if (totalRunning > 0) { + processes.put("stuck_query_count", totalRunning); + processes.put("longest_query_seconds", longestQuerySeconds); + processes.put("severity", totalRunning > 3 ? "CRITICAL" : "WARNING"); + processes.put("queries", stuckQueries); + metrics.put("stuck_processes", processes); + logger.warn("MySQL: {} stuck processes detected, longest running for {} seconds", totalRunning, longestQuerySeconds); + } else { + processes.put("stuck_query_count", 0); + processes.put("longest_query_seconds", 0); + metrics.put("stuck_processes", processes); + } + } + } catch (Exception e) { + logger.debug("Could not check stuck processes (might not support PROCESSLIST)", e); + } + } + + private void checkInnoDBStatus(Connection connection, Map metrics) { + try (PreparedStatement stmt = connection.prepareStatement( + "SELECT OBJECT_SCHEMA, OBJECT_NAME, ENGINE_LOCK_ID, ENGINE_TRANSACTION_ID, THREAD_ID, EVENT_ID, OBJECT_INSTANCE_BEGIN, LOCK_TYPE, LOCK_MODE, LOCK_STATUS, SOURCE, OWNER_THREAD_ID, OWNER_EVENT_ID " + + "FROM INFORMATION_SCHEMA.INNODB_LOCKS")) { + try (ResultSet rs = stmt.executeQuery()) { + Map innodb = new LinkedHashMap<>(); + int waitingLocks = 0; + int grantedLocks = 0; + + while (rs.next()) { + String lockStatus = rs.getString("LOCK_STATUS"); + if ("WAITING".equals(lockStatus)) { + waitingLocks++; + } else { + grantedLocks++; + } + } + + // Check for deadlocks + try (PreparedStatement deadlockStmt = connection.prepareStatement( + "SHOW ENGINE INNODB STATUS")) { + try (ResultSet dlRs = deadlockStmt.executeQuery()) { + if (dlRs.next()) { + String status = dlRs.getString(3); + int deadlockCount = (status != null && status.contains("DEADLOCK")) ? 1 : 0; + innodb.put("deadlocks_detected", deadlockCount); + } + } + } catch (Exception e) { + logger.debug("Could not check InnoDB deadlocks", e); + } + + innodb.put("active_transactions", grantedLocks); + innodb.put("waiting_on_locks", waitingLocks); + if (waitingLocks > 0) { + innodb.put("severity", waitingLocks > 2 ? "CRITICAL" : "WARNING"); + } + + metrics.put("innodb", innodb); + + if (waitingLocks > 0) { + logger.warn("MySQL: {} transactions waiting on InnoDB locks", waitingLocks); + } + } + } catch (Exception e) { + logger.debug("Could not check InnoDB status (might not support INNODB_LOCKS or not using InnoDB)", e); + } + } + + private Map getMySQLStatusVariables(Connection connection) { + Map statusVars = new LinkedHashMap<>(); + try (PreparedStatement stmt = connection.prepareStatement("SHOW STATUS")) { + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + statusVars.put(rs.getString("Variable_name"), rs.getString("Value")); + } + } + } catch (Exception e) { + logger.debug("Could not retrieve MySQL status variables", e); + } + return statusVars; + } private Map checkRedisHealth(boolean includeDetails) { Map details = new LinkedHashMap<>(); details.put("type", "Redis"); - - if (includeDetails) { - details.put("host", redisHost); - details.put("port", redisPort); - } return performHealthCheck("Redis", details, () -> { try { @@ -219,8 +473,7 @@ private Map checkRedisHealth(boolean includeDetails) { ).get(REDIS_TIMEOUT_SECONDS, TimeUnit.SECONDS); if ("PONG".equals(pong)) { - String version = includeDetails ? getRedisVersionWithTimeout() : null; - return new HealthCheckResult(true, version, null); + return new HealthCheckResult(true, null, null); } return new HealthCheckResult(false, null, "Ping returned unexpected response"); } catch (TimeoutException e) { @@ -237,11 +490,6 @@ private Map checkRedisHealth(boolean includeDetails) { private Map checkElasticsearchHealth(boolean includeDetails) { Map details = new LinkedHashMap<>(); details.put("type", ELASTICSEARCH_TYPE); - - if (includeDetails) { - details.put("host", elasticsearchHost); - details.put("port", elasticsearchPort); - } return performHealthCheck(ELASTICSEARCH_TYPE, details, () -> { if (!elasticsearchClientReady || elasticsearchRestClient == null) { @@ -305,9 +553,6 @@ private void buildHealthyStatus(Map status, Map String componentName, long responseTime, HealthCheckResult result) { logger.debug("{} health check: UP ({}ms)", componentName, responseTime); status.put(STATUS_KEY, STATUS_UP); - if (result.version != null) { - details.put("version", result.version); - } } private void buildUnhealthyStatus(Map status, Map details, @@ -315,9 +560,7 @@ private void buildUnhealthyStatus(Map status, Map buildExceptionStatus(Map status, Map status.put(STATUS_KEY, STATUS_DOWN); details.put("responseTimeMs", responseTime); - if (includeDetails) { - details.put("error", errorMessage != null ? errorMessage : "Health check failed"); - } - details.put("errorType", "InternalError"); + details.put("error", errorMessage != null ? errorMessage : "Health check failed"); status.put("details", details); return status; From d70184d29fdf778ccc720ba03067cd98932a6a3d Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Thu, 19 Feb 2026 10:12:37 +0530 Subject: [PATCH 08/20] refactor(health): simplify MySQL health check and remove sensitive details --- .../controller/health/HealthController.java | 51 +- .../service/health/HealthService.java | 512 ++---------------- 2 files changed, 53 insertions(+), 510 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/controller/health/HealthController.java b/src/main/java/com/iemr/common/identity/controller/health/HealthController.java index ef06de0..f682295 100644 --- a/src/main/java/com/iemr/common/identity/controller/health/HealthController.java +++ b/src/main/java/com/iemr/common/identity/controller/health/HealthController.java @@ -66,8 +66,7 @@ public ResponseEntity> checkHealth(HttpServletRequest reques logger.info("Health check endpoint called"); try { - // Always include detailed metrics in health response - Map healthStatus = healthService.checkHealth(true); + Map healthStatus = healthService.checkHealth(); String overallStatus = (String) healthStatus.get("status"); // Return 200 if overall status is UP, 503 if DOWN @@ -82,58 +81,10 @@ public ResponseEntity> checkHealth(HttpServletRequest reques // Return sanitized error response Map errorResponse = Map.of( "status", "DOWN", - "error", "Health check service unavailable", "timestamp", Instant.now().toString() ); return new ResponseEntity<>(errorResponse, HttpStatus.SERVICE_UNAVAILABLE); } } - - private boolean isUserAuthenticated(HttpServletRequest request) { - String token = null; - - // First, try to get token from JwtToken header - token = request.getHeader("JwtToken"); - - // If not found, try Authorization header - if (token == null || token.trim().isEmpty()) { - String authHeader = request.getHeader("Authorization"); - if (authHeader != null && !authHeader.trim().isEmpty()) { - // Extract token from "Bearer " format - token = authHeader.startsWith("Bearer ") - ? authHeader.substring(7) - : authHeader; - } - } - - // If still not found, try to get from cookies - if (token == null || token.trim().isEmpty()) { - token = getJwtTokenFromCookies(request); - } - - // Validate the token if found - if (token != null && !token.trim().isEmpty()) { - try { - return jwtAuthenticationUtil.validateUserIdAndJwtToken(token); - } catch (Exception e) { - logger.debug("JWT token validation failed: {}", e.getMessage()); - return false; - } - } - - return false; - } - - private String getJwtTokenFromCookies(HttpServletRequest request) { - Cookie[] cookies = request.getCookies(); - if (cookies != null) { - for (Cookie cookie : cookies) { - if (cookie.getName().equalsIgnoreCase("Jwttoken")) { - return cookie.getValue(); - } - } - } - return null; - } } diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java index 5e38b87..53a29f0 100644 --- a/src/main/java/com/iemr/common/identity/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -55,19 +55,14 @@ public class HealthService { private static final Logger logger = LoggerFactory.getLogger(HealthService.class); private static final String STATUS_KEY = "status"; private static final String DB_HEALTH_CHECK_QUERY = "SELECT 1 as health_check"; - private static final String DB_VERSION_QUERY = "SELECT VERSION()"; private static final String STATUS_UP = "UP"; private static final String STATUS_DOWN = "DOWN"; - private static final String UNKNOWN_VALUE = "unknown"; private static final String ELASTICSEARCH_TYPE = "Elasticsearch"; private static final int REDIS_TIMEOUT_SECONDS = 3; private final DataSource dataSource; private final ExecutorService executorService = Executors.newFixedThreadPool(4); private final RedisTemplate redisTemplate; - private final String dbUrl; - private final String redisHost; - private final int redisPort; private final String elasticsearchHost; private final int elasticsearchPort; private final boolean elasticsearchEnabled; @@ -76,17 +71,11 @@ public class HealthService { public HealthService(DataSource dataSource, @Autowired(required = false) RedisTemplate redisTemplate, - @Value("${spring.datasource.url:unknown}") String dbUrl, - @Value("${spring.data.redis.host:localhost}") String redisHost, - @Value("${spring.data.redis.port:6379}") int redisPort, @Value("${elasticsearch.host:localhost}") String elasticsearchHost, @Value("${elasticsearch.port:9200}") int elasticsearchPort, @Value("${elasticsearch.enabled:false}") boolean elasticsearchEnabled) { this.dataSource = dataSource; this.redisTemplate = redisTemplate; - this.dbUrl = dbUrl; - this.redisHost = redisHost; - this.redisPort = redisPort; this.elasticsearchHost = elasticsearchHost; this.elasticsearchPort = elasticsearchPort; this.elasticsearchEnabled = elasticsearchEnabled; @@ -131,19 +120,19 @@ private void initializeElasticsearchClient() { } } - public Map checkHealth(boolean includeDetails) { + public Map checkHealth() { Map healthStatus = new LinkedHashMap<>(); Map components = new LinkedHashMap<>(); boolean overallHealth = true; - Map mysqlStatus = checkMySQLHealth(includeDetails); + Map mysqlStatus = checkMySQLHealth(); components.put("mysql", mysqlStatus); if (!isHealthy(mysqlStatus)) { overallHealth = false; } if (redisTemplate != null) { - Map redisStatus = checkRedisHealth(includeDetails); + Map redisStatus = checkRedisHealth(); components.put("redis", redisStatus); if (!isHealthy(redisStatus)) { overallHealth = false; @@ -151,7 +140,7 @@ public Map checkHealth(boolean includeDetails) { } if (elasticsearchEnabled && elasticsearchClientReady) { - Map elasticsearchStatus = checkElasticsearchHealth(includeDetails); + Map elasticsearchStatus = checkElasticsearchHealth(); components.put("elasticsearch", elasticsearchStatus); if (!isHealthy(elasticsearchStatus)) { overallHealth = false; @@ -166,305 +155,34 @@ public Map checkHealth(boolean includeDetails) { return healthStatus; } - public Map checkHealth() { - return checkHealth(true); - } - - private Map checkMySQLHealth(boolean includeDetails) { - Map details = new LinkedHashMap<>(); - details.put("type", "MySQL"); - - return performHealthCheck("MySQL", details, () -> { + private Map checkMySQLHealth() { + Map status = new LinkedHashMap<>(); + + return performHealthCheck("MySQL", status, () -> { try { try (Connection connection = dataSource.getConnection()) { - if (connection.isValid(2)) { - try (PreparedStatement stmt = connection.prepareStatement(DB_HEALTH_CHECK_QUERY)) { - stmt.setQueryTimeout(3); - try (ResultSet rs = stmt.executeQuery()) { - if (rs.next() && rs.getInt(1) == 1) { - return new HealthCheckResult(true, null, null); - } + try (PreparedStatement stmt = connection.prepareStatement(DB_HEALTH_CHECK_QUERY)) { + stmt.setQueryTimeout(3); + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next() && rs.getInt(1) == 1) { + return new HealthCheckResult(true, null); } } } - return new HealthCheckResult(false, null, "Connection validation failed"); + return new HealthCheckResult(false, "Query validation failed"); } } catch (Exception e) { - throw new IllegalStateException("MySQL connection failed: " + e.getMessage(), e); - } - }, includeDetails); - } - - private void addAdvancedMySQLMetrics(Connection connection, Map details) { - try { - // Only add advanced metrics for MySQL, not for H2 or other databases - if (!isMySQLDatabase(connection)) { - logger.debug("Advanced metrics only supported for MySQL, skipping for this database"); - return; - } - - Map metrics = new LinkedHashMap<>(); - - // Get server status variables (uptime, connections, etc.) - Map statusVars = getMySQLStatusVariables(connection); - - if (!statusVars.isEmpty()) { - // Connections - Map connections = new LinkedHashMap<>(); - String threads = statusVars.get("Threads_connected"); - String maxConnections = statusVars.get("max_connections"); - if (threads != null && maxConnections != null) { - connections.put("active", Integer.parseInt(threads)); - connections.put("max", Integer.parseInt(maxConnections)); - try { - int active = Integer.parseInt(threads); - int max = Integer.parseInt(maxConnections); - connections.put("usage_percent", (active * 100) / max); - } catch (NumberFormatException e) { - logger.debug("Could not calculate connection usage percent"); - } - } - if (!connections.isEmpty()) { - metrics.put("connections", connections); - } - - // Uptime - String uptime = statusVars.get("Uptime"); - if (uptime != null) { - try { - long uptimeSeconds = Long.parseLong(uptime); - metrics.put("uptime_seconds", uptimeSeconds); - metrics.put("uptime_hours", uptimeSeconds / 3600); - } catch (NumberFormatException e) { - logger.debug("Could not parse uptime"); - } - } - - // Slow queries - String slowQueries = statusVars.get("Slow_queries"); - if (slowQueries != null) { - metrics.put("slow_queries", Integer.parseInt(slowQueries)); - } - - // Questions (total queries) - String questions = statusVars.get("Questions"); - if (questions != null) { - metrics.put("total_queries", Long.parseLong(questions)); - } - } - - // Database size - Map database = new LinkedHashMap<>(); - try (PreparedStatement stmt = connection.prepareStatement( - "SELECT table_schema, ROUND(SUM(data_length + index_length) / 1024 / 1024, 2) as size_mb " + - "FROM information_schema.tables WHERE table_schema = database() GROUP BY table_schema")) { - try (ResultSet rs = stmt.executeQuery()) { - if (rs.next()) { - double sizeMb = rs.getDouble("size_mb"); - database.put("size_mb", sizeMb); - } - } - } catch (Exception e) { - logger.debug("Could not retrieve database size", e); - } - - // Table count - try (PreparedStatement stmt = connection.prepareStatement( - "SELECT COUNT(*) as table_count FROM information_schema.tables WHERE table_schema = database()")) { - try (ResultSet rs = stmt.executeQuery()) { - if (rs.next()) { - database.put("tables_count", rs.getInt("table_count")); - } - } - } catch (Exception e) { - logger.debug("Could not retrieve table count", e); - } - - if (!database.isEmpty()) { - metrics.put("database", database); + throw new IllegalStateException("MySQL health check failed: " + e.getMessage(), e); } - - // Add deep health checks (locks, stuck processes, deadlocks) - addDeepMySQLHealthChecks(connection, metrics); - - if (!metrics.isEmpty()) { - details.put("metrics", metrics); - } - - } catch (Exception e) { - logger.debug("Error retrieving advanced MySQL metrics", e); - } + }); } - private boolean isMySQLDatabase(Connection connection) { - try { - String databaseProductName = connection.getMetaData().getDatabaseProductName(); - return databaseProductName != null && databaseProductName.toLowerCase().contains("mysql"); - } catch (Exception e) { - logger.debug("Could not determine database type", e); - return false; - } - } - - private void addDeepMySQLHealthChecks(Connection connection, Map metrics) { - // Check for table locks - checkTableLocks(connection, metrics); - - // Check for stuck/long-running queries - checkStuckProcesses(connection, metrics); - - // Check for InnoDB deadlocks - checkInnoDBStatus(connection, metrics); - } - - private void checkTableLocks(Connection connection, Map metrics) { - try (PreparedStatement stmt = connection.prepareStatement( - "SELECT OBJECT_SCHEMA, OBJECT_NAME, COUNT(*) as lock_count " + - "FROM INFORMATION_SCHEMA.METADATA_LOCKS " + - "WHERE OBJECT_SCHEMA != 'mysql' AND OBJECT_SCHEMA != 'information_schema' " + - "GROUP BY OBJECT_SCHEMA, OBJECT_NAME")) { - try (ResultSet rs = stmt.executeQuery()) { - Map locks = new LinkedHashMap<>(); - int totalLocks = 0; - java.util.List lockedTables = new java.util.ArrayList<>(); - - while (rs.next()) { - totalLocks += rs.getInt("lock_count"); - String tableName = rs.getString("OBJECT_SCHEMA") + "." + rs.getString("OBJECT_NAME"); - lockedTables.add(tableName); - } - - if (totalLocks > 0) { - locks.put("is_locked", true); - locks.put("locked_tables_count", lockedTables.size()); - locks.put("locked_tables", lockedTables); - locks.put("total_locks", totalLocks); - locks.put("severity", totalLocks > 5 ? "CRITICAL" : "WARNING"); - metrics.put("table_locks", locks); - logger.warn("MySQL: {} table locks detected on {} tables", totalLocks, lockedTables.size()); - } else { - locks.put("is_locked", false); - locks.put("locked_tables_count", 0); - metrics.put("table_locks", locks); - } - } - } catch (Exception e) { - logger.debug("Could not check table locks (might not support METADATA_LOCKS)", e); - } - } - - private void checkStuckProcesses(Connection connection, Map metrics) { - try (PreparedStatement stmt = connection.prepareStatement( - "SELECT ID, USER, HOST, DB, TIME, COMMAND, STATE, INFO " + - "FROM INFORMATION_SCHEMA.PROCESSLIST " + - "WHERE COMMAND != 'Sleep' AND TIME > 300")) { // Processes running > 5 minutes - try (ResultSet rs = stmt.executeQuery()) { - Map processes = new LinkedHashMap<>(); - java.util.List> stuckQueries = new java.util.ArrayList<>(); - int totalRunning = 0; - long longestQuerySeconds = 0; - - while (rs.next()) { - long queryTime = rs.getLong("TIME"); - longestQuerySeconds = Math.max(longestQuerySeconds, queryTime); - - Map query = new LinkedHashMap<>(); - query.put("id", rs.getInt("ID")); - query.put("user", rs.getString("USER")); - query.put("command", rs.getString("COMMAND")); - query.put("time_seconds", queryTime); - query.put("state", rs.getString("STATE")); - - stuckQueries.add(query); - totalRunning++; - } - - if (totalRunning > 0) { - processes.put("stuck_query_count", totalRunning); - processes.put("longest_query_seconds", longestQuerySeconds); - processes.put("severity", totalRunning > 3 ? "CRITICAL" : "WARNING"); - processes.put("queries", stuckQueries); - metrics.put("stuck_processes", processes); - logger.warn("MySQL: {} stuck processes detected, longest running for {} seconds", totalRunning, longestQuerySeconds); - } else { - processes.put("stuck_query_count", 0); - processes.put("longest_query_seconds", 0); - metrics.put("stuck_processes", processes); - } - } - } catch (Exception e) { - logger.debug("Could not check stuck processes (might not support PROCESSLIST)", e); - } - } - - private void checkInnoDBStatus(Connection connection, Map metrics) { - try (PreparedStatement stmt = connection.prepareStatement( - "SELECT OBJECT_SCHEMA, OBJECT_NAME, ENGINE_LOCK_ID, ENGINE_TRANSACTION_ID, THREAD_ID, EVENT_ID, OBJECT_INSTANCE_BEGIN, LOCK_TYPE, LOCK_MODE, LOCK_STATUS, SOURCE, OWNER_THREAD_ID, OWNER_EVENT_ID " + - "FROM INFORMATION_SCHEMA.INNODB_LOCKS")) { - try (ResultSet rs = stmt.executeQuery()) { - Map innodb = new LinkedHashMap<>(); - int waitingLocks = 0; - int grantedLocks = 0; - - while (rs.next()) { - String lockStatus = rs.getString("LOCK_STATUS"); - if ("WAITING".equals(lockStatus)) { - waitingLocks++; - } else { - grantedLocks++; - } - } - - // Check for deadlocks - try (PreparedStatement deadlockStmt = connection.prepareStatement( - "SHOW ENGINE INNODB STATUS")) { - try (ResultSet dlRs = deadlockStmt.executeQuery()) { - if (dlRs.next()) { - String status = dlRs.getString(3); - int deadlockCount = (status != null && status.contains("DEADLOCK")) ? 1 : 0; - innodb.put("deadlocks_detected", deadlockCount); - } - } - } catch (Exception e) { - logger.debug("Could not check InnoDB deadlocks", e); - } - - innodb.put("active_transactions", grantedLocks); - innodb.put("waiting_on_locks", waitingLocks); - if (waitingLocks > 0) { - innodb.put("severity", waitingLocks > 2 ? "CRITICAL" : "WARNING"); - } - - metrics.put("innodb", innodb); - - if (waitingLocks > 0) { - logger.warn("MySQL: {} transactions waiting on InnoDB locks", waitingLocks); - } - } - } catch (Exception e) { - logger.debug("Could not check InnoDB status (might not support INNODB_LOCKS or not using InnoDB)", e); - } - } - - private Map getMySQLStatusVariables(Connection connection) { - Map statusVars = new LinkedHashMap<>(); - try (PreparedStatement stmt = connection.prepareStatement("SHOW STATUS")) { - try (ResultSet rs = stmt.executeQuery()) { - while (rs.next()) { - statusVars.put(rs.getString("Variable_name"), rs.getString("Value")); - } - } - } catch (Exception e) { - logger.debug("Could not retrieve MySQL status variables", e); - } - return statusVars; - } - private Map checkRedisHealth(boolean includeDetails) { - Map details = new LinkedHashMap<>(); - details.put("type", "Redis"); - return performHealthCheck("Redis", details, () -> { + private Map checkRedisHealth() { + Map status = new LinkedHashMap<>(); + + return performHealthCheck("Redis", status, () -> { try { // Wrap PING in CompletableFuture with timeout String pong = CompletableFuture.supplyAsync(() -> @@ -473,28 +191,27 @@ private Map checkRedisHealth(boolean includeDetails) { ).get(REDIS_TIMEOUT_SECONDS, TimeUnit.SECONDS); if ("PONG".equals(pong)) { - return new HealthCheckResult(true, null, null); + return new HealthCheckResult(true, null); } - return new HealthCheckResult(false, null, "Ping returned unexpected response"); + return new HealthCheckResult(false, "Ping returned unexpected response"); } catch (TimeoutException e) { - return new HealthCheckResult(false, null, "Redis ping timed out after " + REDIS_TIMEOUT_SECONDS + " seconds"); + return new HealthCheckResult(false, "Redis ping timed out after " + REDIS_TIMEOUT_SECONDS + " seconds"); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - return new HealthCheckResult(false, null, "Redis health check was interrupted"); + return new HealthCheckResult(false, "Redis health check was interrupted"); } catch (Exception e) { throw new IllegalStateException("Redis health check failed", e); } - }, includeDetails); + }); } - private Map checkElasticsearchHealth(boolean includeDetails) { - Map details = new LinkedHashMap<>(); - details.put("type", ELASTICSEARCH_TYPE); + private Map checkElasticsearchHealth() { + Map status = new LinkedHashMap<>(); - return performHealthCheck(ELASTICSEARCH_TYPE, details, () -> { + return performHealthCheck(ELASTICSEARCH_TYPE, status, () -> { if (!elasticsearchClientReady || elasticsearchRestClient == null) { logger.debug("Elasticsearch RestClient not ready"); - return new HealthCheckResult(false, null, "Elasticsearch client not ready"); + return new HealthCheckResult(false, "Elasticsearch client not ready"); } try { @@ -505,192 +222,67 @@ private Map checkElasticsearchHealth(boolean includeDetails) { int statusCode = response.getStatusLine().getStatusCode(); if (statusCode == 200) { logger.debug("{} health check successful", ELASTICSEARCH_TYPE); - return new HealthCheckResult(true, null, null); + return new HealthCheckResult(true, null); } - return new HealthCheckResult(false, null, "HTTP " + statusCode); + return new HealthCheckResult(false, "HTTP " + statusCode); } catch (java.net.ConnectException e) { logger.error("{} connection refused on {}:{}", ELASTICSEARCH_TYPE, elasticsearchHost, elasticsearchPort, e); - return new HealthCheckResult(false, null, "Connection refused"); + return new HealthCheckResult(false, "Connection refused"); } catch (java.io.IOException e) { logger.error("{} IO error: {}", ELASTICSEARCH_TYPE, e.getMessage(), e); - return new HealthCheckResult(false, null, "IO Error: " + e.getMessage()); + return new HealthCheckResult(false, "IO Error: " + e.getMessage()); } catch (Exception e) { logger.error("{} error: {} - {}", ELASTICSEARCH_TYPE, e.getClass().getSimpleName(), e.getMessage(), e); - return new HealthCheckResult(false, null, e.getMessage()); + return new HealthCheckResult(false, e.getMessage()); } - }, includeDetails); + }); } private Map performHealthCheck(String componentName, - Map details, - Supplier checker, - boolean includeDetails) { - Map status = new LinkedHashMap<>(); + Map status, + Supplier checker) { long startTime = System.currentTimeMillis(); try { HealthCheckResult result = checker.get(); long responseTime = System.currentTimeMillis() - startTime; - details.put("responseTimeMs", responseTime); + status.put("responseTimeMs", responseTime); if (result.isHealthy) { - buildHealthyStatus(status, details, componentName, responseTime, result); + logger.debug("{} health check: UP ({}ms)", componentName, responseTime); + status.put(STATUS_KEY, STATUS_UP); } else { - buildUnhealthyStatus(status, details, componentName, result, includeDetails); + String safeError = result.error != null ? result.error : "Health check failed"; + logger.warn("{} health check failed: {}", componentName, safeError); + status.put(STATUS_KEY, STATUS_DOWN); } - status.put("details", details); return status; } catch (Exception e) { long responseTime = System.currentTimeMillis() - startTime; - return buildExceptionStatus(status, details, componentName, e, includeDetails, responseTime); + logger.error("{} health check failed with exception: {}", componentName, e.getMessage(), e); + + String errorMessage = e.getCause() != null ? e.getCause().getMessage() : e.getMessage(); + + status.put(STATUS_KEY, STATUS_DOWN); + status.put("responseTimeMs", responseTime); + + return status; } } - private void buildHealthyStatus(Map status, Map details, - String componentName, long responseTime, HealthCheckResult result) { - logger.debug("{} health check: UP ({}ms)", componentName, responseTime); - status.put(STATUS_KEY, STATUS_UP); - } - - private void buildUnhealthyStatus(Map status, Map details, - String componentName, HealthCheckResult result, boolean includeDetails) { - String safeError = result.error != null ? result.error : "Health check failed"; - logger.warn("{} health check failed: {}", componentName, safeError); - status.put(STATUS_KEY, STATUS_DOWN); - details.put("error", safeError); - details.put("errorType", "CheckFailed"); - } - - private Map buildExceptionStatus(Map status, Map details, - String componentName, Exception e, boolean includeDetails, long responseTime) { - logger.error("{} health check failed with exception: {}", componentName, e.getMessage(), e); - - String errorMessage = e.getCause() != null ? e.getCause().getMessage() : e.getMessage(); - - status.put(STATUS_KEY, STATUS_DOWN); - details.put("responseTimeMs", responseTime); - details.put("error", errorMessage != null ? errorMessage : "Health check failed"); - status.put("details", details); - - return status; - } - private boolean isHealthy(Map componentStatus) { return STATUS_UP.equals(componentStatus.get(STATUS_KEY)); } - private String getMySQLVersion(Connection connection) { - try (PreparedStatement stmt = connection.prepareStatement(DB_VERSION_QUERY); - ResultSet rs = stmt.executeQuery()) { - if (rs.next()) { - return rs.getString(1); - } - } catch (Exception e) { - logger.debug("Could not retrieve MySQL version", e); - } - return null; - } - - private String getRedisVersion() { - try { - Properties info = redisTemplate.execute((RedisCallback) connection -> - connection.serverCommands().info("server") - ); - if (info != null && info.containsKey("redis_version")) { - return info.getProperty("redis_version"); - } - } catch (Exception e) { - logger.debug("Could not retrieve Redis version", e); - } - return null; - } - - private String getRedisVersionWithTimeout() { - try { - return CompletableFuture.supplyAsync(this::getRedisVersion, executorService) - .get(REDIS_TIMEOUT_SECONDS, TimeUnit.SECONDS); - } catch (TimeoutException e) { - logger.debug("Redis version retrieval timed out"); - return null; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.debug("Redis version retrieval was interrupted"); - return null; - } catch (Exception e) { - logger.debug("Could not retrieve Redis version with timeout", e); - return null; - } - } - - - - private String extractHost(String jdbcUrl) { - if (jdbcUrl == null || UNKNOWN_VALUE.equals(jdbcUrl)) { - return UNKNOWN_VALUE; - } - try { - String withoutPrefix = jdbcUrl.replaceFirst("jdbc:mysql://", ""); - int slashIndex = withoutPrefix.indexOf('/'); - String hostPort = slashIndex > 0 - ? withoutPrefix.substring(0, slashIndex) - : withoutPrefix; - int colonIndex = hostPort.indexOf(':'); - return colonIndex > 0 ? hostPort.substring(0, colonIndex) : hostPort; - } catch (Exception e) { - logger.debug("Could not extract host from URL", e); - } - return UNKNOWN_VALUE; - } - - private String extractPort(String jdbcUrl) { - if (jdbcUrl == null || UNKNOWN_VALUE.equals(jdbcUrl)) { - return UNKNOWN_VALUE; - } - try { - String withoutPrefix = jdbcUrl.replaceFirst("jdbc:mysql://", ""); - int slashIndex = withoutPrefix.indexOf('/'); - String hostPort = slashIndex > 0 - ? withoutPrefix.substring(0, slashIndex) - : withoutPrefix; - int colonIndex = hostPort.indexOf(':'); - return colonIndex > 0 ? hostPort.substring(colonIndex + 1) : "3306"; - } catch (Exception e) { - logger.debug("Could not extract port from URL", e); - } - return "3306"; - } - - private String extractDatabaseName(String jdbcUrl) { - if (jdbcUrl == null || UNKNOWN_VALUE.equals(jdbcUrl)) { - return UNKNOWN_VALUE; - } - try { - int lastSlash = jdbcUrl.lastIndexOf('/'); - if (lastSlash >= 0 && lastSlash < jdbcUrl.length() - 1) { - String afterSlash = jdbcUrl.substring(lastSlash + 1); - int queryStart = afterSlash.indexOf('?'); - if (queryStart > 0) { - return afterSlash.substring(0, queryStart); - } - return afterSlash; - } - } catch (Exception e) { - logger.debug("Could not extract database name from URL", e); - } - return UNKNOWN_VALUE; - } - private static class HealthCheckResult { final boolean isHealthy; - final String version; final String error; - HealthCheckResult(boolean isHealthy, String version, String error) { + HealthCheckResult(boolean isHealthy, String error) { this.isHealthy = isHealthy; - this.version = version; this.error = error; } } From d130a8562d03336fea9105be4089cb506fd0a59c Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Thu, 19 Feb 2026 10:15:46 +0530 Subject: [PATCH 09/20] fix(health): remove unused imports and variables --- .../com/iemr/common/identity/service/health/HealthService.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java index 53a29f0..ecdb5b3 100644 --- a/src/main/java/com/iemr/common/identity/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -29,7 +29,6 @@ import java.time.Instant; import java.util.LinkedHashMap; import java.util.Map; -import java.util.Properties; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -264,8 +263,6 @@ private Map performHealthCheck(String componentName, long responseTime = System.currentTimeMillis() - startTime; logger.error("{} health check failed with exception: {}", componentName, e.getMessage(), e); - String errorMessage = e.getCause() != null ? e.getCause().getMessage() : e.getMessage(); - status.put(STATUS_KEY, STATUS_DOWN); status.put("responseTimeMs", responseTime); From 3e755eb79f8634f982c3c8648a6a1c5d7dfeebae Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Thu, 19 Feb 2026 10:40:33 +0530 Subject: [PATCH 10/20] refactor(health): address nitpicks (configurable ES scheme, log noise, graceful shutdown, record) --- .../service/health/HealthService.java | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java index ecdb5b3..91a56f8 100644 --- a/src/main/java/com/iemr/common/identity/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -64,6 +64,7 @@ public class HealthService { private final RedisTemplate redisTemplate; private final String elasticsearchHost; private final int elasticsearchPort; + private final String elasticsearchScheme; private final boolean elasticsearchEnabled; private RestClient elasticsearchRestClient; private boolean elasticsearchClientReady = false; @@ -72,11 +73,13 @@ public HealthService(DataSource dataSource, @Autowired(required = false) RedisTemplate redisTemplate, @Value("${elasticsearch.host:localhost}") String elasticsearchHost, @Value("${elasticsearch.port:9200}") int elasticsearchPort, + @Value("${elasticsearch.scheme:http}") String elasticsearchScheme, @Value("${elasticsearch.enabled:false}") boolean elasticsearchEnabled) { this.dataSource = dataSource; this.redisTemplate = redisTemplate; this.elasticsearchHost = elasticsearchHost; this.elasticsearchPort = elasticsearchPort; + this.elasticsearchScheme = elasticsearchScheme; this.elasticsearchEnabled = elasticsearchEnabled; } @@ -90,7 +93,19 @@ public void init() { @jakarta.annotation.PreDestroy public void cleanup() { - executorService.shutdownNow(); + // Gracefully shutdown executor service + executorService.shutdown(); + try { + if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) { + logger.warn("Executor service did not terminate within 5 seconds, forcing shutdown"); + executorService.shutdownNow(); + } + } catch (InterruptedException e) { + logger.warn("Interrupted while waiting for executor service termination"); + executorService.shutdownNow(); + Thread.currentThread().interrupt(); + } + if (elasticsearchRestClient != null) { try { elasticsearchRestClient.close(); @@ -103,7 +118,7 @@ public void cleanup() { private void initializeElasticsearchClient() { try { this.elasticsearchRestClient = RestClient.builder( - new HttpHost(elasticsearchHost, elasticsearchPort, "http") + new HttpHost(elasticsearchHost, elasticsearchPort, elasticsearchScheme) ) .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder @@ -149,7 +164,7 @@ public Map checkHealth() { healthStatus.put(STATUS_KEY, overallHealth ? STATUS_UP : STATUS_DOWN); healthStatus.put("timestamp", Instant.now().toString()); healthStatus.put("components", components); - logger.info("Health check completed - Overall status: {}", overallHealth ? STATUS_UP : STATUS_DOWN); + logger.debug("Health check completed - Overall status: {}", overallHealth ? STATUS_UP : STATUS_DOWN); return healthStatus; } @@ -248,11 +263,11 @@ private Map performHealthCheck(String componentName, status.put("responseTimeMs", responseTime); - if (result.isHealthy) { + if (result.isHealthy()) { logger.debug("{} health check: UP ({}ms)", componentName, responseTime); status.put(STATUS_KEY, STATUS_UP); } else { - String safeError = result.error != null ? result.error : "Health check failed"; + String safeError = result.error() != null ? result.error() : "Health check failed"; logger.warn("{} health check failed: {}", componentName, safeError); status.put(STATUS_KEY, STATUS_DOWN); } @@ -274,13 +289,5 @@ private boolean isHealthy(Map componentStatus) { return STATUS_UP.equals(componentStatus.get(STATUS_KEY)); } - private static class HealthCheckResult { - final boolean isHealthy; - final String error; - - HealthCheckResult(boolean isHealthy, String error) { - this.isHealthy = isHealthy; - this.error = error; - } - } + private static record HealthCheckResult(boolean isHealthy, String error) {} } From d74ee262a2d471e7f62de4e59667058faa302cc5 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sat, 21 Feb 2026 12:18:51 +0530 Subject: [PATCH 11/20] fix(health): scope PROCESSLIST lock-wait check to application DB user --- .../controller/health/HealthController.java | 8 +- .../service/health/HealthService.java | 498 +++++++++++++++--- 2 files changed, 426 insertions(+), 80 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/controller/health/HealthController.java b/src/main/java/com/iemr/common/identity/controller/health/HealthController.java index f682295..b7cc364 100644 --- a/src/main/java/com/iemr/common/identity/controller/health/HealthController.java +++ b/src/main/java/com/iemr/common/identity/controller/health/HealthController.java @@ -59,18 +59,18 @@ public HealthController(HealthService healthService, JwtAuthenticationUtil jwtAu @Operation(summary = "Check infrastructure health", description = "Returns the health status of MySQL, Redis, Elasticsearch, and other configured services") @ApiResponses({ - @ApiResponse(responseCode = "200", description = "All checked components are UP"), + @ApiResponse(responseCode = "200", description = "Services are UP or DEGRADED (operational with warnings)"), @ApiResponse(responseCode = "503", description = "One or more critical services are DOWN") }) - public ResponseEntity> checkHealth(HttpServletRequest request) { + public ResponseEntity> checkHealth() { logger.info("Health check endpoint called"); try { Map healthStatus = healthService.checkHealth(); String overallStatus = (String) healthStatus.get("status"); - // Return 200 if overall status is UP, 503 if DOWN - HttpStatus httpStatus = "UP".equals(overallStatus) ? HttpStatus.OK : HttpStatus.SERVICE_UNAVAILABLE; + // Return 503 only if DOWN; 200 for both UP and DEGRADED (DEGRADED = operational with warnings) + HttpStatus httpStatus = "DOWN".equals(overallStatus) ? HttpStatus.SERVICE_UNAVAILABLE : HttpStatus.OK; logger.debug("Health check completed with status: {}", overallStatus); return new ResponseEntity<>(healthStatus, httpStatus); diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java index 91a56f8..1c13d0f 100644 --- a/src/main/java/com/iemr/common/identity/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -26,16 +26,24 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.Statement; import java.time.Instant; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Properties; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Supplier; import javax.sql.DataSource; +import com.zaxxer.hikari.HikariDataSource; +import com.zaxxer.hikari.HikariPoolMXBean; +import java.lang.management.ManagementFactory; +import javax.management.MBeanServer; +import javax.management.ObjectName; import jakarta.annotation.PostConstruct; import org.apache.http.HttpHost; import org.elasticsearch.client.Request; @@ -54,32 +62,74 @@ public class HealthService { private static final Logger logger = LoggerFactory.getLogger(HealthService.class); private static final String STATUS_KEY = "status"; private static final String DB_HEALTH_CHECK_QUERY = "SELECT 1 as health_check"; + private static final String DB_VERSION_QUERY = "SELECT VERSION()"; private static final String STATUS_UP = "UP"; private static final String STATUS_DOWN = "DOWN"; + private static final String STATUS_DEGRADED = "DEGRADED"; + private static final String UNKNOWN_VALUE = "unknown"; private static final String ELASTICSEARCH_TYPE = "Elasticsearch"; private static final int REDIS_TIMEOUT_SECONDS = 3; + + // Severity levels and keys + private static final String SEVERITY_KEY = "severity"; + private static final String SEVERITY_OK = "OK"; + private static final String SEVERITY_WARNING = "WARNING"; + private static final String SEVERITY_CRITICAL = "CRITICAL"; + + // Performance threshold (milliseconds) - response time > 2000ms = DEGRADED + private static final long RESPONSE_TIME_THRESHOLD_MS = 2000; + + // Advanced checks configuration + private static final long ADVANCED_CHECKS_TIMEOUT_MS = 500; // Strict timeout for advanced checks + private static final long ADVANCED_CHECKS_THROTTLE_SECONDS = 30; // Run at most once per 30 seconds + + // Response keys + private static final String ERROR_KEY = "error"; + private static final String MESSAGE_KEY = "message"; + private static final String RESPONSE_TIME_KEY = "responseTimeMs"; + + // Diagnostic event codes for concise logging + private static final String DIAGNOSTIC_LOCK_WAIT = "MYSQL_LOCK_WAIT"; + private static final String DIAGNOSTIC_DEADLOCK = "MYSQL_DEADLOCK"; + private static final String DIAGNOSTIC_SLOW_QUERIES = "MYSQL_SLOW_QUERIES"; + private static final String DIAGNOSTIC_POOL_EXHAUSTED = "MYSQL_POOL_EXHAUSTED"; + private static final String DIAGNOSTIC_LOG_TEMPLATE = "Diagnostic: {}"; private final DataSource dataSource; private final ExecutorService executorService = Executors.newFixedThreadPool(4); private final RedisTemplate redisTemplate; + private final String dbUrl; + private final String redisHost; + private final int redisPort; private final String elasticsearchHost; private final int elasticsearchPort; - private final String elasticsearchScheme; private final boolean elasticsearchEnabled; private RestClient elasticsearchRestClient; private boolean elasticsearchClientReady = false; + + // Advanced checks throttling (thread-safe) + private volatile long lastAdvancedCheckTime = 0; + private volatile AdvancedCheckResult cachedAdvancedCheckResult = null; + private final ReentrantReadWriteLock advancedCheckLock = new ReentrantReadWriteLock(); + + // Deadlock check resilience - disable after first permission error + private volatile boolean deadlockCheckDisabled = false; public HealthService(DataSource dataSource, @Autowired(required = false) RedisTemplate redisTemplate, + @Value("${spring.datasource.url:unknown}") String dbUrl, + @Value("${spring.data.redis.host:localhost}") String redisHost, + @Value("${spring.data.redis.port:6379}") int redisPort, @Value("${elasticsearch.host:localhost}") String elasticsearchHost, @Value("${elasticsearch.port:9200}") int elasticsearchPort, - @Value("${elasticsearch.scheme:http}") String elasticsearchScheme, @Value("${elasticsearch.enabled:false}") boolean elasticsearchEnabled) { this.dataSource = dataSource; this.redisTemplate = redisTemplate; + this.dbUrl = dbUrl; + this.redisHost = redisHost; + this.redisPort = redisPort; this.elasticsearchHost = elasticsearchHost; this.elasticsearchPort = elasticsearchPort; - this.elasticsearchScheme = elasticsearchScheme; this.elasticsearchEnabled = elasticsearchEnabled; } @@ -93,19 +143,7 @@ public void init() { @jakarta.annotation.PreDestroy public void cleanup() { - // Gracefully shutdown executor service - executorService.shutdown(); - try { - if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) { - logger.warn("Executor service did not terminate within 5 seconds, forcing shutdown"); - executorService.shutdownNow(); - } - } catch (InterruptedException e) { - logger.warn("Interrupted while waiting for executor service termination"); - executorService.shutdownNow(); - Thread.currentThread().interrupt(); - } - + executorService.shutdownNow(); if (elasticsearchRestClient != null) { try { elasticsearchRestClient.close(); @@ -118,7 +156,7 @@ public void cleanup() { private void initializeElasticsearchClient() { try { this.elasticsearchRestClient = RestClient.builder( - new HttpHost(elasticsearchHost, elasticsearchPort, elasticsearchScheme) + new HttpHost(elasticsearchHost, elasticsearchPort, "http") ) .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder @@ -134,19 +172,19 @@ private void initializeElasticsearchClient() { } } - public Map checkHealth() { + public Map checkHealth(boolean includeDetails) { Map healthStatus = new LinkedHashMap<>(); Map components = new LinkedHashMap<>(); boolean overallHealth = true; - Map mysqlStatus = checkMySQLHealth(); + Map mysqlStatus = checkMySQLHealth(includeDetails); components.put("mysql", mysqlStatus); if (!isHealthy(mysqlStatus)) { overallHealth = false; } if (redisTemplate != null) { - Map redisStatus = checkRedisHealth(); + Map redisStatus = checkRedisHealth(includeDetails); components.put("redis", redisStatus); if (!isHealthy(redisStatus)) { overallHealth = false; @@ -154,7 +192,7 @@ public Map checkHealth() { } if (elasticsearchEnabled && elasticsearchClientReady) { - Map elasticsearchStatus = checkElasticsearchHealth(); + Map elasticsearchStatus = checkElasticsearchHealth(includeDetails); components.put("elasticsearch", elasticsearchStatus); if (!isHealthy(elasticsearchStatus)) { overallHealth = false; @@ -164,68 +202,71 @@ public Map checkHealth() { healthStatus.put(STATUS_KEY, overallHealth ? STATUS_UP : STATUS_DOWN); healthStatus.put("timestamp", Instant.now().toString()); healthStatus.put("components", components); - logger.debug("Health check completed - Overall status: {}", overallHealth ? STATUS_UP : STATUS_DOWN); + logger.info("Health check completed - Overall status: {}", overallHealth ? STATUS_UP : STATUS_DOWN); return healthStatus; } - private Map checkMySQLHealth() { - Map status = new LinkedHashMap<>(); - - return performHealthCheck("MySQL", status, () -> { + public Map checkHealth() { + return checkHealth(true); + } + + private Map checkMySQLHealth(boolean includeDetails) { + Map details = new LinkedHashMap<>(); + details.put("type", "MySQL"); + + return performHealthCheck("MySQL", details, () -> { try { try (Connection connection = dataSource.getConnection()) { - try (PreparedStatement stmt = connection.prepareStatement(DB_HEALTH_CHECK_QUERY)) { - stmt.setQueryTimeout(3); - try (ResultSet rs = stmt.executeQuery()) { - if (rs.next() && rs.getInt(1) == 1) { - return new HealthCheckResult(true, null); + if (connection.isValid(2)) { + try (PreparedStatement stmt = connection.prepareStatement(DB_HEALTH_CHECK_QUERY)) { + stmt.setQueryTimeout(3); + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next() && rs.getInt(1) == 1) { + // Basic check passed - run advanced checks with throttling + boolean isDegraded = performAdvancedMySQLChecksWithThrottle(connection); + return new HealthCheckResult(true, null, isDegraded); + } } } } - return new HealthCheckResult(false, "Query validation failed"); + return new HealthCheckResult(false, "Connection validation failed", false); } } catch (Exception e) { - throw new IllegalStateException("MySQL health check failed: " + e.getMessage(), e); + throw new IllegalStateException("MySQL connection failed: " + e.getMessage(), e); } - }); + }, includeDetails); } - private Map checkRedisHealth() { - Map status = new LinkedHashMap<>(); + private Map checkRedisHealth(boolean includeDetails) { + Map details = new LinkedHashMap<>(); + details.put("type", "Redis"); - return performHealthCheck("Redis", status, () -> { + return performHealthCheck("Redis", details, () -> { try { - // Wrap PING in CompletableFuture with timeout - String pong = CompletableFuture.supplyAsync(() -> - redisTemplate.execute((RedisCallback) connection -> connection.ping()), - executorService - ).get(REDIS_TIMEOUT_SECONDS, TimeUnit.SECONDS); + // Run Redis PING synchronously - avoid nested CompletableFuture on same executor + String pong = redisTemplate.execute((RedisCallback) connection -> connection.ping()); if ("PONG".equals(pong)) { - return new HealthCheckResult(true, null); + return new HealthCheckResult(true, null, false); } - return new HealthCheckResult(false, "Ping returned unexpected response"); - } catch (TimeoutException e) { - return new HealthCheckResult(false, "Redis ping timed out after " + REDIS_TIMEOUT_SECONDS + " seconds"); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return new HealthCheckResult(false, "Redis health check was interrupted"); + return new HealthCheckResult(false, "Ping returned unexpected response", false); } catch (Exception e) { throw new IllegalStateException("Redis health check failed", e); } - }); + }, includeDetails); } - private Map checkElasticsearchHealth() { - Map status = new LinkedHashMap<>(); + private Map checkElasticsearchHealth(boolean includeDetails) { + Map details = new LinkedHashMap<>(); + details.put("type", ELASTICSEARCH_TYPE); - return performHealthCheck(ELASTICSEARCH_TYPE, status, () -> { + return performHealthCheck(ELASTICSEARCH_TYPE, details, () -> { if (!elasticsearchClientReady || elasticsearchRestClient == null) { logger.debug("Elasticsearch RestClient not ready"); - return new HealthCheckResult(false, "Elasticsearch client not ready"); + return new HealthCheckResult(false, "Elasticsearch client not ready", false); } try { @@ -236,58 +277,363 @@ private Map checkElasticsearchHealth() { int statusCode = response.getStatusLine().getStatusCode(); if (statusCode == 200) { logger.debug("{} health check successful", ELASTICSEARCH_TYPE); - return new HealthCheckResult(true, null); + return new HealthCheckResult(true, null, false); } - return new HealthCheckResult(false, "HTTP " + statusCode); + return new HealthCheckResult(false, "HTTP " + statusCode, false); } catch (java.net.ConnectException e) { logger.error("{} connection refused on {}:{}", ELASTICSEARCH_TYPE, elasticsearchHost, elasticsearchPort, e); - return new HealthCheckResult(false, "Connection refused"); + return new HealthCheckResult(false, "Connection refused", false); } catch (java.io.IOException e) { logger.error("{} IO error: {}", ELASTICSEARCH_TYPE, e.getMessage(), e); - return new HealthCheckResult(false, "IO Error: " + e.getMessage()); + return new HealthCheckResult(false, "IO Error: " + e.getMessage(), false); } catch (Exception e) { logger.error("{} error: {} - {}", ELASTICSEARCH_TYPE, e.getClass().getSimpleName(), e.getMessage(), e); - return new HealthCheckResult(false, e.getMessage()); + return new HealthCheckResult(false, e.getMessage(), false); } - }); + }, includeDetails); } private Map performHealthCheck(String componentName, - Map status, - Supplier checker) { + Map details, + Supplier checker, + boolean includeDetails) { + Map status = new LinkedHashMap<>(); long startTime = System.currentTimeMillis(); try { HealthCheckResult result = checker.get(); long responseTime = System.currentTimeMillis() - startTime; - status.put("responseTimeMs", responseTime); + details.put("responseTimeMs", responseTime); - if (result.isHealthy()) { - logger.debug("{} health check: UP ({}ms)", componentName, responseTime); - status.put(STATUS_KEY, STATUS_UP); + if (result.isHealthy) { + buildHealthyStatus(status, details, componentName, responseTime, result); } else { - String safeError = result.error() != null ? result.error() : "Health check failed"; - logger.warn("{} health check failed: {}", componentName, safeError); - status.put(STATUS_KEY, STATUS_DOWN); + buildUnhealthyStatus(status, details, componentName, result, includeDetails); } + status.put("details", details); return status; } catch (Exception e) { long responseTime = System.currentTimeMillis() - startTime; - logger.error("{} health check failed with exception: {}", componentName, e.getMessage(), e); - - status.put(STATUS_KEY, STATUS_DOWN); - status.put("responseTimeMs", responseTime); - - return status; + return buildExceptionStatus(status, details, componentName, e, includeDetails, responseTime); + } + } + + private void buildHealthyStatus(Map status, Map details, + String componentName, long responseTime, HealthCheckResult result) { + logger.debug("{} health check: UP ({}ms)", componentName, responseTime); + + // Determine status based on health, response time, and degradation flags + String statusValue = result.isDegraded ? STATUS_DEGRADED : STATUS_UP; + status.put(STATUS_KEY, statusValue); + + String severity = determineSeverity(result.isHealthy, responseTime, result.isDegraded); + status.put(SEVERITY_KEY, severity); + + // Include message if there's an error (informational when healthy) + if (result.error != null) { + status.put(MESSAGE_KEY, result.error); } } + private void buildUnhealthyStatus(Map status, Map details, + String componentName, HealthCheckResult result, boolean includeDetails) { + String safeError = result.error != null ? result.error : "Health check failed"; + logger.warn("{} health check failed: {}", componentName, safeError); + status.put(STATUS_KEY, STATUS_DOWN); + status.put(SEVERITY_KEY, SEVERITY_CRITICAL); + details.put("error", safeError); + details.put("errorType", "CheckFailed"); + } + + private Map buildExceptionStatus(Map status, Map details, + String componentName, Exception e, boolean includeDetails, long responseTime) { + logger.error("{} health check failed with exception: {}", componentName, e.getMessage(), e); + + String errorMessage = e.getCause() != null ? e.getCause().getMessage() : e.getMessage(); + + status.put(STATUS_KEY, STATUS_DOWN); + status.put(SEVERITY_KEY, SEVERITY_CRITICAL); + details.put("responseTimeMs", responseTime); + details.put("error", errorMessage != null ? errorMessage : "Health check failed"); + status.put("details", details); + + return status; + } + private String determineSeverity(boolean isHealthy, long responseTimeMs, boolean isDegraded) { + if (!isHealthy) { + return SEVERITY_CRITICAL; + } + + if (isDegraded) { + return SEVERITY_WARNING; + } + + if (responseTimeMs > RESPONSE_TIME_THRESHOLD_MS) { + return SEVERITY_WARNING; + } + + return SEVERITY_OK; + } + private boolean isHealthy(Map componentStatus) { return STATUS_UP.equals(componentStatus.get(STATUS_KEY)); } - private static record HealthCheckResult(boolean isHealthy, String error) {} + private String getMySQLVersion(Connection connection) { + try (PreparedStatement stmt = connection.prepareStatement(DB_VERSION_QUERY); + ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + return rs.getString(1); + } + } catch (Exception e) { + logger.debug("Could not retrieve MySQL version", e); + } + return null; + } + + private String getRedisVersion() { + try { + Properties info = redisTemplate.execute((RedisCallback) connection -> + connection.serverCommands().info("server") + ); + if (info != null && info.containsKey("redis_version")) { + return info.getProperty("redis_version"); + } + } catch (Exception e) { + logger.debug("Could not retrieve Redis version", e); + } + return null; + } + + private String getRedisVersionWithTimeout() { + try { + return CompletableFuture.supplyAsync(this::getRedisVersion, executorService) + .get(REDIS_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } catch (TimeoutException e) { + logger.debug("Redis version retrieval timed out"); + return null; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.debug("Redis version retrieval was interrupted"); + return null; + } catch (Exception e) { + logger.debug("Could not retrieve Redis version with timeout", e); + return null; + } + } + + private boolean performAdvancedMySQLChecksWithThrottle(Connection connection) { + long currentTime = System.currentTimeMillis(); + + advancedCheckLock.readLock().lock(); + try { + if (cachedAdvancedCheckResult != null && + (currentTime - lastAdvancedCheckTime) < ADVANCED_CHECKS_THROTTLE_SECONDS * 1000) { + return cachedAdvancedCheckResult.isDegraded; + } + } finally { + advancedCheckLock.readLock().unlock(); + } + + advancedCheckLock.writeLock().lock(); + try { + currentTime = System.currentTimeMillis(); + // Double-check after acquiring write lock + if (cachedAdvancedCheckResult != null && + (currentTime - lastAdvancedCheckTime) < ADVANCED_CHECKS_THROTTLE_SECONDS * 1000) { + return cachedAdvancedCheckResult.isDegraded; + } + + AdvancedCheckResult result = performAdvancedMySQLChecks(connection); + + // Cache the result + lastAdvancedCheckTime = currentTime; + cachedAdvancedCheckResult = result; + + return result.isDegraded; + } finally { + advancedCheckLock.writeLock().unlock(); + } + } + + private AdvancedCheckResult performAdvancedMySQLChecks(Connection connection) { + try { + boolean hasIssues = false; + + if (hasLockWaits(connection)) { + logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_LOCK_WAIT); + hasIssues = true; + } + + if (hasDeadlocks(connection)) { + logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_DEADLOCK); + hasIssues = true; + } + + if (hasSlowQueries(connection)) { + logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_SLOW_QUERIES); + hasIssues = true; + } + + if (hasConnectionPoolExhaustion()) { + logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_POOL_EXHAUSTED); + hasIssues = true; + } + + return new AdvancedCheckResult(hasIssues); + } catch (Exception e) { + logger.debug("Advanced MySQL checks encountered exception, marking degraded"); + return new AdvancedCheckResult(true); + } + } + + private boolean hasLockWaits(Connection connection) { + try (PreparedStatement stmt = connection.prepareStatement( + "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + + "WHERE (state = 'Waiting for table metadata lock' " + + " OR state = 'Waiting for row lock' " + + " OR state = 'Waiting for lock') " + + "AND user = USER()")) { + stmt.setQueryTimeout(2); + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + int lockCount = rs.getInt(1); + return lockCount > 0; + } + } + } catch (Exception e) { + logger.debug("Could not check for lock waits"); + } + return false; + } + + private boolean hasDeadlocks(Connection connection) { + // Skip deadlock check if already disabled due to permissions + if (deadlockCheckDisabled) { + return false; + } + + try (PreparedStatement stmt = connection.prepareStatement("SHOW ENGINE INNODB STATUS")) { + stmt.setQueryTimeout(2); + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + String innodbStatus = rs.getString(3); + return innodbStatus != null && innodbStatus.contains("LATEST DETECTED DEADLOCK"); + } + } + } catch (java.sql.SQLException e) { + // Check if this is a permission error + if (e.getMessage() != null && + (e.getMessage().contains("Access denied") || + e.getMessage().contains("permission"))) { + // Disable this check permanently after first permission error + deadlockCheckDisabled = true; + logger.warn("Deadlock check disabled: Insufficient privileges"); + } else { + logger.debug("Could not check for deadlocks"); + } + } catch (Exception e) { + logger.debug("Could not check for deadlocks"); + } + return false; + } + + private boolean hasSlowQueries(Connection connection) { + try (PreparedStatement stmt = connection.prepareStatement( + "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + + "WHERE command != 'Sleep' AND time > ? AND user NOT IN ('event_scheduler', 'system user')")) { + stmt.setQueryTimeout(2); + stmt.setInt(1, 10); // Queries running longer than 10 seconds + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + int slowQueryCount = rs.getInt(1); + return slowQueryCount > 3; // Alert if more than 3 slow queries + } + } + } catch (Exception e) { + logger.debug("Could not check for slow queries"); + } + return false; + } + + private boolean hasConnectionPoolExhaustion() { + // Use HikariCP metrics if available + if (dataSource instanceof HikariDataSource hikariDataSource) { + try { + HikariPoolMXBean poolMXBean = hikariDataSource.getHikariPoolMXBean(); + + if (poolMXBean != null) { + int activeConnections = poolMXBean.getActiveConnections(); + int maxPoolSize = hikariDataSource.getMaximumPoolSize(); + + // Alert if > 80% of pool is exhausted + int threshold = (int) (maxPoolSize * 0.8); + return activeConnections > threshold; + } + } catch (Exception e) { + logger.debug("Could not retrieve HikariCP pool metrics"); + } + } + + // Fallback: try to get pool metrics via JMX if HikariCP is not directly available + return checkPoolMetricsViaJMX(); + } + + private boolean checkPoolMetricsViaJMX() { + try { + MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer(); + ObjectName objectName = new ObjectName("com.zaxxer.hikari:type=Pool (*)"); + var mBeans = mBeanServer.queryMBeans(objectName, null); + + for (var mBean : mBeans) { + if (evaluatePoolMetrics(mBeanServer, mBean.getObjectName())) { + return true; + } + } + } catch (Exception e) { + logger.debug("Could not access HikariCP pool metrics via JMX"); + } + + // No pool metrics available - disable this check + logger.debug("Pool exhaustion check disabled: HikariCP metrics unavailable"); + return false; + } + + private boolean evaluatePoolMetrics(MBeanServer mBeanServer, ObjectName objectName) { + try { + Integer activeConnections = (Integer) mBeanServer.getAttribute(objectName, "ActiveConnections"); + Integer maximumPoolSize = (Integer) mBeanServer.getAttribute(objectName, "MaximumPoolSize"); + + if (activeConnections != null && maximumPoolSize != null) { + int threshold = (int) (maximumPoolSize * 0.8); + return activeConnections > threshold; + } + } catch (Exception e) { + // Continue to next MBean + } + return false; + } + + private static class AdvancedCheckResult { + final boolean isDegraded; + + AdvancedCheckResult(boolean isDegraded) { + this.isDegraded = isDegraded; + } + } + + private static class HealthCheckResult { + final boolean isHealthy; + final String error; + final boolean isDegraded; + + HealthCheckResult(boolean isHealthy, String error, boolean isDegraded) { + this.isHealthy = isHealthy; + this.error = error; + this.isDegraded = isDegraded; + } + } } From 79ce1928fe06de2f47879255c1c79d41f62c94ab Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sat, 21 Feb 2026 12:28:41 +0530 Subject: [PATCH 12/20] refactor(health): remove unused params and reuse response/error constants --- .../service/health/HealthService.java | 73 +++++-------------- 1 file changed, 19 insertions(+), 54 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java index 1c13d0f..0bff75a 100644 --- a/src/main/java/com/iemr/common/identity/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -26,7 +26,6 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.sql.Statement; import java.time.Instant; import java.util.LinkedHashMap; import java.util.Map; @@ -75,13 +74,8 @@ public class HealthService { private static final String SEVERITY_OK = "OK"; private static final String SEVERITY_WARNING = "WARNING"; private static final String SEVERITY_CRITICAL = "CRITICAL"; - - // Performance threshold (milliseconds) - response time > 2000ms = DEGRADED private static final long RESPONSE_TIME_THRESHOLD_MS = 2000; - - // Advanced checks configuration - private static final long ADVANCED_CHECKS_TIMEOUT_MS = 500; // Strict timeout for advanced checks - private static final long ADVANCED_CHECKS_THROTTLE_SECONDS = 30; // Run at most once per 30 seconds + private static final long ADVANCED_CHECKS_THROTTLE_SECONDS = 30; // Response keys private static final String ERROR_KEY = "error"; @@ -177,14 +171,14 @@ public Map checkHealth(boolean includeDetails) { Map components = new LinkedHashMap<>(); boolean overallHealth = true; - Map mysqlStatus = checkMySQLHealth(includeDetails); + Map mysqlStatus = checkMySQLHealth(); components.put("mysql", mysqlStatus); if (!isHealthy(mysqlStatus)) { overallHealth = false; } if (redisTemplate != null) { - Map redisStatus = checkRedisHealth(includeDetails); + Map redisStatus = checkRedisHealth(); components.put("redis", redisStatus); if (!isHealthy(redisStatus)) { overallHealth = false; @@ -192,7 +186,7 @@ public Map checkHealth(boolean includeDetails) { } if (elasticsearchEnabled && elasticsearchClientReady) { - Map elasticsearchStatus = checkElasticsearchHealth(includeDetails); + Map elasticsearchStatus = checkElasticsearchHealth(); components.put("elasticsearch", elasticsearchStatus); if (!isHealthy(elasticsearchStatus)) { overallHealth = false; @@ -211,7 +205,7 @@ public Map checkHealth() { return checkHealth(true); } - private Map checkMySQLHealth(boolean includeDetails) { + private Map checkMySQLHealth() { Map details = new LinkedHashMap<>(); details.put("type", "MySQL"); @@ -235,12 +229,12 @@ private Map checkMySQLHealth(boolean includeDetails) { } catch (Exception e) { throw new IllegalStateException("MySQL connection failed: " + e.getMessage(), e); } - }, includeDetails); + }); } - private Map checkRedisHealth(boolean includeDetails) { + private Map checkRedisHealth() { Map details = new LinkedHashMap<>(); details.put("type", "Redis"); @@ -256,10 +250,10 @@ private Map checkRedisHealth(boolean includeDetails) { } catch (Exception e) { throw new IllegalStateException("Redis health check failed", e); } - }, includeDetails); + }); } - private Map checkElasticsearchHealth(boolean includeDetails) { + private Map checkElasticsearchHealth() { Map details = new LinkedHashMap<>(); details.put("type", ELASTICSEARCH_TYPE); @@ -290,13 +284,12 @@ private Map checkElasticsearchHealth(boolean includeDetails) { logger.error("{} error: {} - {}", ELASTICSEARCH_TYPE, e.getClass().getSimpleName(), e.getMessage(), e); return new HealthCheckResult(false, e.getMessage(), false); } - }, includeDetails); + }); } private Map performHealthCheck(String componentName, Map details, - Supplier checker, - boolean includeDetails) { + Supplier checker) { Map status = new LinkedHashMap<>(); long startTime = System.currentTimeMillis(); @@ -304,12 +297,12 @@ private Map performHealthCheck(String componentName, HealthCheckResult result = checker.get(); long responseTime = System.currentTimeMillis() - startTime; - details.put("responseTimeMs", responseTime); + details.put(RESPONSE_TIME_KEY, responseTime); if (result.isHealthy) { buildHealthyStatus(status, details, componentName, responseTime, result); } else { - buildUnhealthyStatus(status, details, componentName, result, includeDetails); + buildUnhealthyStatus(status, details, componentName, result); } status.put("details", details); @@ -317,7 +310,7 @@ private Map performHealthCheck(String componentName, } catch (Exception e) { long responseTime = System.currentTimeMillis() - startTime; - return buildExceptionStatus(status, details, componentName, e, includeDetails, responseTime); + return buildExceptionStatus(status, details, componentName, e, responseTime); } } @@ -339,25 +332,25 @@ private void buildHealthyStatus(Map status, Map } private void buildUnhealthyStatus(Map status, Map details, - String componentName, HealthCheckResult result, boolean includeDetails) { + String componentName, HealthCheckResult result) { String safeError = result.error != null ? result.error : "Health check failed"; logger.warn("{} health check failed: {}", componentName, safeError); status.put(STATUS_KEY, STATUS_DOWN); status.put(SEVERITY_KEY, SEVERITY_CRITICAL); - details.put("error", safeError); + details.put(ERROR_KEY, safeError); details.put("errorType", "CheckFailed"); } private Map buildExceptionStatus(Map status, Map details, - String componentName, Exception e, boolean includeDetails, long responseTime) { + String componentName, Exception e, long responseTime) { logger.error("{} health check failed with exception: {}", componentName, e.getMessage(), e); String errorMessage = e.getCause() != null ? e.getCause().getMessage() : e.getMessage(); status.put(STATUS_KEY, STATUS_DOWN); status.put(SEVERITY_KEY, SEVERITY_CRITICAL); - details.put("responseTimeMs", responseTime); - details.put("error", errorMessage != null ? errorMessage : "Health check failed"); + details.put(RESPONSE_TIME_KEY, responseTime); + details.put(ERROR_KEY, errorMessage != null ? errorMessage : "Health check failed"); status.put("details", details); return status; @@ -382,18 +375,6 @@ private boolean isHealthy(Map componentStatus) { return STATUS_UP.equals(componentStatus.get(STATUS_KEY)); } - private String getMySQLVersion(Connection connection) { - try (PreparedStatement stmt = connection.prepareStatement(DB_VERSION_QUERY); - ResultSet rs = stmt.executeQuery()) { - if (rs.next()) { - return rs.getString(1); - } - } catch (Exception e) { - logger.debug("Could not retrieve MySQL version", e); - } - return null; - } - private String getRedisVersion() { try { Properties info = redisTemplate.execute((RedisCallback) connection -> @@ -408,22 +389,6 @@ private String getRedisVersion() { return null; } - private String getRedisVersionWithTimeout() { - try { - return CompletableFuture.supplyAsync(this::getRedisVersion, executorService) - .get(REDIS_TIMEOUT_SECONDS, TimeUnit.SECONDS); - } catch (TimeoutException e) { - logger.debug("Redis version retrieval timed out"); - return null; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.debug("Redis version retrieval was interrupted"); - return null; - } catch (Exception e) { - logger.debug("Could not retrieve Redis version with timeout", e); - return null; - } - } private boolean performAdvancedMySQLChecksWithThrottle(Connection connection) { long currentTime = System.currentTimeMillis(); From c24c710049474d3289d570697991e9f885959986 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sat, 21 Feb 2026 12:34:04 +0530 Subject: [PATCH 13/20] fix(health): remove unused imports and methods --- .../service/health/HealthService.java | 27 +++---------------- 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java index 0bff75a..0020094 100644 --- a/src/main/java/com/iemr/common/identity/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -30,11 +30,8 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Properties; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Supplier; import javax.sql.DataSource; @@ -166,7 +163,7 @@ private void initializeElasticsearchClient() { } } - public Map checkHealth(boolean includeDetails) { + public Map checkHealth() { Map healthStatus = new LinkedHashMap<>(); Map components = new LinkedHashMap<>(); boolean overallHealth = true; @@ -201,10 +198,6 @@ public Map checkHealth(boolean includeDetails) { return healthStatus; } - public Map checkHealth() { - return checkHealth(true); - } - private Map checkMySQLHealth() { Map details = new LinkedHashMap<>(); details.put("type", "MySQL"); @@ -300,7 +293,7 @@ private Map performHealthCheck(String componentName, details.put(RESPONSE_TIME_KEY, responseTime); if (result.isHealthy) { - buildHealthyStatus(status, details, componentName, responseTime, result); + buildHealthyStatus(status, componentName, responseTime, result); } else { buildUnhealthyStatus(status, details, componentName, result); } @@ -314,7 +307,7 @@ private Map performHealthCheck(String componentName, } } - private void buildHealthyStatus(Map status, Map details, + private void buildHealthyStatus(Map status, String componentName, long responseTime, HealthCheckResult result) { logger.debug("{} health check: UP ({}ms)", componentName, responseTime); @@ -375,20 +368,6 @@ private boolean isHealthy(Map componentStatus) { return STATUS_UP.equals(componentStatus.get(STATUS_KEY)); } - private String getRedisVersion() { - try { - Properties info = redisTemplate.execute((RedisCallback) connection -> - connection.serverCommands().info("server") - ); - if (info != null && info.containsKey("redis_version")) { - return info.getProperty("redis_version"); - } - } catch (Exception e) { - logger.debug("Could not retrieve Redis version", e); - } - return null; - } - private boolean performAdvancedMySQLChecksWithThrottle(Connection connection) { long currentTime = System.currentTimeMillis(); From baf29a194a372d0e96d646b3a3da987c61a47a85 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sat, 21 Feb 2026 12:37:26 +0530 Subject: [PATCH 14/20] chore(health): clean up unused imports, params, and dead helpers --- .../com/iemr/common/identity/service/health/HealthService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java index 0020094..3da20a7 100644 --- a/src/main/java/com/iemr/common/identity/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -29,7 +29,6 @@ import java.time.Instant; import java.util.LinkedHashMap; import java.util.Map; -import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.locks.ReentrantReadWriteLock; From 474c913d220988d061a48056d5ebc81778d3b3bf Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sun, 22 Feb 2026 10:15:01 +0530 Subject: [PATCH 15/20] fix(health): avoid sharing JDBC connections across threads in advanced MySQL checks --- .../service/health/HealthService.java | 115 +++++++++--------- 1 file changed, 55 insertions(+), 60 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java index 3da20a7..4e76d95 100644 --- a/src/main/java/com/iemr/common/identity/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -57,11 +57,9 @@ public class HealthService { private static final Logger logger = LoggerFactory.getLogger(HealthService.class); private static final String STATUS_KEY = "status"; private static final String DB_HEALTH_CHECK_QUERY = "SELECT 1 as health_check"; - private static final String DB_VERSION_QUERY = "SELECT VERSION()"; private static final String STATUS_UP = "UP"; private static final String STATUS_DOWN = "DOWN"; private static final String STATUS_DEGRADED = "DEGRADED"; - private static final String UNKNOWN_VALUE = "unknown"; private static final String ELASTICSEARCH_TYPE = "Elasticsearch"; private static final int REDIS_TIMEOUT_SECONDS = 3; @@ -78,19 +76,23 @@ public class HealthService { private static final String MESSAGE_KEY = "message"; private static final String RESPONSE_TIME_KEY = "responseTimeMs"; + // Component name constants + private static final String MYSQL_COMPONENT = "MySQL"; + private static final String REDIS_COMPONENT = "Redis"; + + // Advanced checks timeout + private static final long ADVANCED_CHECKS_TIMEOUT_MS = 500L; + // Diagnostic event codes for concise logging private static final String DIAGNOSTIC_LOCK_WAIT = "MYSQL_LOCK_WAIT"; - private static final String DIAGNOSTIC_DEADLOCK = "MYSQL_DEADLOCK"; private static final String DIAGNOSTIC_SLOW_QUERIES = "MYSQL_SLOW_QUERIES"; private static final String DIAGNOSTIC_POOL_EXHAUSTED = "MYSQL_POOL_EXHAUSTED"; private static final String DIAGNOSTIC_LOG_TEMPLATE = "Diagnostic: {}"; private final DataSource dataSource; private final ExecutorService executorService = Executors.newFixedThreadPool(4); + private final ExecutorService advancedCheckExecutor; private final RedisTemplate redisTemplate; - private final String dbUrl; - private final String redisHost; - private final int redisPort; private final String elasticsearchHost; private final int elasticsearchPort; private final boolean elasticsearchEnabled; @@ -101,23 +103,19 @@ public class HealthService { private volatile long lastAdvancedCheckTime = 0; private volatile AdvancedCheckResult cachedAdvancedCheckResult = null; private final ReentrantReadWriteLock advancedCheckLock = new ReentrantReadWriteLock(); - - // Deadlock check resilience - disable after first permission error - private volatile boolean deadlockCheckDisabled = false; public HealthService(DataSource dataSource, @Autowired(required = false) RedisTemplate redisTemplate, - @Value("${spring.datasource.url:unknown}") String dbUrl, - @Value("${spring.data.redis.host:localhost}") String redisHost, - @Value("${spring.data.redis.port:6379}") int redisPort, @Value("${elasticsearch.host:localhost}") String elasticsearchHost, @Value("${elasticsearch.port:9200}") int elasticsearchPort, @Value("${elasticsearch.enabled:false}") boolean elasticsearchEnabled) { this.dataSource = dataSource; + this.advancedCheckExecutor = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "health-advanced-check"); + t.setDaemon(true); + return t; + }); this.redisTemplate = redisTemplate; - this.dbUrl = dbUrl; - this.redisHost = redisHost; - this.redisPort = redisPort; this.elasticsearchHost = elasticsearchHost; this.elasticsearchPort = elasticsearchPort; this.elasticsearchEnabled = elasticsearchEnabled; @@ -134,6 +132,7 @@ public void init() { @jakarta.annotation.PreDestroy public void cleanup() { executorService.shutdownNow(); + advancedCheckExecutor.shutdownNow(); if (elasticsearchRestClient != null) { try { elasticsearchRestClient.close(); @@ -199,9 +198,9 @@ public Map checkHealth() { private Map checkMySQLHealth() { Map details = new LinkedHashMap<>(); - details.put("type", "MySQL"); + details.put("type", MYSQL_COMPONENT); - return performHealthCheck("MySQL", details, () -> { + return performHealthCheck(MYSQL_COMPONENT, details, () -> { try { try (Connection connection = dataSource.getConnection()) { if (connection.isValid(2)) { @@ -210,7 +209,7 @@ private Map checkMySQLHealth() { try (ResultSet rs = stmt.executeQuery()) { if (rs.next() && rs.getInt(1) == 1) { // Basic check passed - run advanced checks with throttling - boolean isDegraded = performAdvancedMySQLChecksWithThrottle(connection); + boolean isDegraded = performAdvancedMySQLChecksWithThrottle(); return new HealthCheckResult(true, null, isDegraded); } } @@ -219,7 +218,7 @@ private Map checkMySQLHealth() { return new HealthCheckResult(false, "Connection validation failed", false); } } catch (Exception e) { - throw new IllegalStateException("MySQL connection failed: " + e.getMessage(), e); + throw new IllegalStateException(MYSQL_COMPONENT + " connection failed: " + e.getMessage(), e); } }); } @@ -368,7 +367,7 @@ private boolean isHealthy(Map componentStatus) { } - private boolean performAdvancedMySQLChecksWithThrottle(Connection connection) { + private boolean performAdvancedMySQLChecksWithThrottle() { long currentTime = System.currentTimeMillis(); advancedCheckLock.readLock().lock(); @@ -390,7 +389,7 @@ private boolean performAdvancedMySQLChecksWithThrottle(Connection connection) { return cachedAdvancedCheckResult.isDegraded; } - AdvancedCheckResult result = performAdvancedMySQLChecks(connection); + AdvancedCheckResult result = performAdvancedMySQLChecks(); // Cache the result lastAdvancedCheckTime = currentTime; @@ -402,7 +401,40 @@ private boolean performAdvancedMySQLChecksWithThrottle(Connection connection) { } } - private AdvancedCheckResult performAdvancedMySQLChecks(Connection connection) { + private AdvancedCheckResult performAdvancedMySQLChecks() { + try { + try (Connection connection = dataSource.getConnection()) { + java.util.concurrent.CompletableFuture future = + java.util.concurrent.CompletableFuture.supplyAsync( + () -> performAdvancedCheckLogic(connection), + advancedCheckExecutor + ); + + return future.get(ADVANCED_CHECKS_TIMEOUT_MS, java.util.concurrent.TimeUnit.MILLISECONDS); + } catch (java.util.concurrent.TimeoutException e) { + logger.debug("Advanced checks timeout, marking degraded"); + return new AdvancedCheckResult(true); + } catch (java.util.concurrent.ExecutionException e) { + if (e.getCause() instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + logger.debug("Advanced checks execution failed, marking degraded"); + return new AdvancedCheckResult(true); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.debug("Advanced checks interrupted, marking degraded"); + return new AdvancedCheckResult(true); + } catch (Exception e) { + logger.debug("Advanced checks encountered exception, marking degraded"); + return new AdvancedCheckResult(true); + } + } catch (Exception e) { + logger.debug("Failed to get connection for advanced checks: {}", e.getMessage()); + return new AdvancedCheckResult(true); + } + } + + private AdvancedCheckResult performAdvancedCheckLogic(Connection connection) { try { boolean hasIssues = false; @@ -411,11 +443,6 @@ private AdvancedCheckResult performAdvancedMySQLChecks(Connection connection) { hasIssues = true; } - if (hasDeadlocks(connection)) { - logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_DEADLOCK); - hasIssues = true; - } - if (hasSlowQueries(connection)) { logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_SLOW_QUERIES); hasIssues = true; @@ -428,7 +455,7 @@ private AdvancedCheckResult performAdvancedMySQLChecks(Connection connection) { return new AdvancedCheckResult(hasIssues); } catch (Exception e) { - logger.debug("Advanced MySQL checks encountered exception, marking degraded"); + logger.debug("Advanced check logic encountered exception"); return new AdvancedCheckResult(true); } } @@ -452,38 +479,6 @@ private boolean hasLockWaits(Connection connection) { } return false; } - - private boolean hasDeadlocks(Connection connection) { - // Skip deadlock check if already disabled due to permissions - if (deadlockCheckDisabled) { - return false; - } - - try (PreparedStatement stmt = connection.prepareStatement("SHOW ENGINE INNODB STATUS")) { - stmt.setQueryTimeout(2); - try (ResultSet rs = stmt.executeQuery()) { - if (rs.next()) { - String innodbStatus = rs.getString(3); - return innodbStatus != null && innodbStatus.contains("LATEST DETECTED DEADLOCK"); - } - } - } catch (java.sql.SQLException e) { - // Check if this is a permission error - if (e.getMessage() != null && - (e.getMessage().contains("Access denied") || - e.getMessage().contains("permission"))) { - // Disable this check permanently after first permission error - deadlockCheckDisabled = true; - logger.warn("Deadlock check disabled: Insufficient privileges"); - } else { - logger.debug("Could not check for deadlocks"); - } - } catch (Exception e) { - logger.debug("Could not check for deadlocks"); - } - return false; - } - private boolean hasSlowQueries(Connection connection) { try (PreparedStatement stmt = connection.prepareStatement( "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + From 618a21b43d030c7c3011b806751c53a87f7dd48e Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sun, 22 Feb 2026 11:59:36 +0530 Subject: [PATCH 16/20] refactor(health): reuse REDIS_COMPONENT constant and extract nested try block --- .../service/health/HealthService.java | 56 ++++++++++--------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java index 4e76d95..b29fb52 100644 --- a/src/main/java/com/iemr/common/identity/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -227,9 +227,9 @@ private Map checkMySQLHealth() { private Map checkRedisHealth() { Map details = new LinkedHashMap<>(); - details.put("type", "Redis"); + details.put("type", REDIS_COMPONENT); - return performHealthCheck("Redis", details, () -> { + return performHealthCheck(REDIS_COMPONENT, details, () -> { try { // Run Redis PING synchronously - avoid nested CompletableFuture on same executor String pong = redisTemplate.execute((RedisCallback) connection -> connection.ping()); @@ -404,29 +404,7 @@ private boolean performAdvancedMySQLChecksWithThrottle() { private AdvancedCheckResult performAdvancedMySQLChecks() { try { try (Connection connection = dataSource.getConnection()) { - java.util.concurrent.CompletableFuture future = - java.util.concurrent.CompletableFuture.supplyAsync( - () -> performAdvancedCheckLogic(connection), - advancedCheckExecutor - ); - - return future.get(ADVANCED_CHECKS_TIMEOUT_MS, java.util.concurrent.TimeUnit.MILLISECONDS); - } catch (java.util.concurrent.TimeoutException e) { - logger.debug("Advanced checks timeout, marking degraded"); - return new AdvancedCheckResult(true); - } catch (java.util.concurrent.ExecutionException e) { - if (e.getCause() instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - logger.debug("Advanced checks execution failed, marking degraded"); - return new AdvancedCheckResult(true); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.debug("Advanced checks interrupted, marking degraded"); - return new AdvancedCheckResult(true); - } catch (Exception e) { - logger.debug("Advanced checks encountered exception, marking degraded"); - return new AdvancedCheckResult(true); + return executeAdvancedCheckAsync(connection); } } catch (Exception e) { logger.debug("Failed to get connection for advanced checks: {}", e.getMessage()); @@ -434,6 +412,34 @@ private AdvancedCheckResult performAdvancedMySQLChecks() { } } + private AdvancedCheckResult executeAdvancedCheckAsync(Connection connection) { + try { + java.util.concurrent.CompletableFuture future = + java.util.concurrent.CompletableFuture.supplyAsync( + () -> performAdvancedCheckLogic(connection), + advancedCheckExecutor + ); + + return future.get(ADVANCED_CHECKS_TIMEOUT_MS, java.util.concurrent.TimeUnit.MILLISECONDS); + } catch (java.util.concurrent.TimeoutException e) { + logger.debug("Advanced checks timeout, marking degraded"); + return new AdvancedCheckResult(true); + } catch (java.util.concurrent.ExecutionException e) { + if (e.getCause() instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + logger.debug("Advanced checks execution failed, marking degraded"); + return new AdvancedCheckResult(true); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.debug("Advanced checks interrupted, marking degraded"); + return new AdvancedCheckResult(true); + } catch (Exception e) { + logger.debug("Advanced checks encountered exception, marking degraded"); + return new AdvancedCheckResult(true); + } + } + private AdvancedCheckResult performAdvancedCheckLogic(Connection connection) { try { boolean hasIssues = false; From 3e7f50ec7b1d08e3c85a116ecf0c0029b739bf5c Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sun, 22 Feb 2026 21:02:07 +0530 Subject: [PATCH 17/20] fix(health): avoid blocking DB I/O under write lock and restore interrupt flag --- .../service/health/HealthService.java | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java index b29fb52..a13ac1a 100644 --- a/src/main/java/com/iemr/common/identity/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -31,6 +31,8 @@ import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ExecutionException; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Supplier; import javax.sql.DataSource; @@ -90,7 +92,7 @@ public class HealthService { private static final String DIAGNOSTIC_LOG_TEMPLATE = "Diagnostic: {}"; private final DataSource dataSource; - private final ExecutorService executorService = Executors.newFixedThreadPool(4); + private final ExecutorService executorService = Executors.newFixedThreadPool(6); private final ExecutorService advancedCheckExecutor; private final RedisTemplate redisTemplate; private final String elasticsearchHost; @@ -103,6 +105,9 @@ public class HealthService { private volatile long lastAdvancedCheckTime = 0; private volatile AdvancedCheckResult cachedAdvancedCheckResult = null; private final ReentrantReadWriteLock advancedCheckLock = new ReentrantReadWriteLock(); + + // Advanced checks always enabled + private static final boolean ADVANCED_HEALTH_CHECKS_ENABLED = true; public HealthService(DataSource dataSource, @Autowired(required = false) RedisTemplate redisTemplate, @@ -368,6 +373,10 @@ private boolean isHealthy(Map componentStatus) { private boolean performAdvancedMySQLChecksWithThrottle() { + if (!ADVANCED_HEALTH_CHECKS_ENABLED) { + return false; // Advanced checks disabled + } + long currentTime = System.currentTimeMillis(); advancedCheckLock.readLock().lock(); @@ -414,17 +423,16 @@ private AdvancedCheckResult performAdvancedMySQLChecks() { private AdvancedCheckResult executeAdvancedCheckAsync(Connection connection) { try { - java.util.concurrent.CompletableFuture future = - java.util.concurrent.CompletableFuture.supplyAsync( - () -> performAdvancedCheckLogic(connection), - advancedCheckExecutor + Future future = + advancedCheckExecutor.submit( + () -> performAdvancedCheckLogic(connection) ); return future.get(ADVANCED_CHECKS_TIMEOUT_MS, java.util.concurrent.TimeUnit.MILLISECONDS); } catch (java.util.concurrent.TimeoutException e) { logger.debug("Advanced checks timeout, marking degraded"); return new AdvancedCheckResult(true); - } catch (java.util.concurrent.ExecutionException e) { + } catch (ExecutionException e) { if (e.getCause() instanceof InterruptedException) { Thread.currentThread().interrupt(); } @@ -488,7 +496,7 @@ private boolean hasLockWaits(Connection connection) { private boolean hasSlowQueries(Connection connection) { try (PreparedStatement stmt = connection.prepareStatement( "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + - "WHERE command != 'Sleep' AND time > ? AND user NOT IN ('event_scheduler', 'system user')")) { + "WHERE command != 'Sleep' AND time > ? AND user = SUBSTRING_INDEX(USER(), '@', 1)")) { stmt.setQueryTimeout(2); stmt.setInt(1, 10); // Queries running longer than 10 seconds try (ResultSet rs = stmt.executeQuery()) { From b05cf7b570464c865ce571e13fc5c0b570076529 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sun, 22 Feb 2026 21:57:22 +0530 Subject: [PATCH 18/20] fix(health): cancel in-flight futures on generic failure --- .../service/health/HealthService.java | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java index a13ac1a..6cda78d 100644 --- a/src/main/java/com/iemr/common/identity/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -33,6 +33,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Supplier; import javax.sql.DataSource; @@ -105,6 +106,7 @@ public class HealthService { private volatile long lastAdvancedCheckTime = 0; private volatile AdvancedCheckResult cachedAdvancedCheckResult = null; private final ReentrantReadWriteLock advancedCheckLock = new ReentrantReadWriteLock(); + private final AtomicBoolean advancedCheckInProgress = new AtomicBoolean(false); // Advanced checks always enabled private static final boolean ADVANCED_HEALTH_CHECKS_ENABLED = true; @@ -387,24 +389,31 @@ private boolean performAdvancedMySQLChecksWithThrottle() { } } finally { advancedCheckLock.readLock().unlock(); + // Only one thread may submit; others fall back to the (stale) cache + if (!advancedCheckInProgress.compareAndSet(false, true)) { + advancedCheckLock.readLock().lock(); + try { + return cachedAdvancedCheckResult != null && cachedAdvancedCheckResult.isDegraded; + } finally { + advancedCheckLock.readLock().unlock(); + } } - advancedCheckLock.writeLock().lock(); try { - currentTime = System.currentTimeMillis(); - // Double-check after acquiring write lock - if (cachedAdvancedCheckResult != null && - (currentTime - lastAdvancedCheckTime) < ADVANCED_CHECKS_THROTTLE_SECONDS * 1000) { - return cachedAdvancedCheckResult.isDegraded; - } - + // DB I/O outside the lock to prevent lock contention AdvancedCheckResult result = performAdvancedMySQLChecks(); - // Cache the result - lastAdvancedCheckTime = currentTime; - cachedAdvancedCheckResult = result; - - return result.isDegraded; + // Re-acquire write lock only for atomic cache update + advancedCheckLock.writeLock().lock(); + try { + lastAdvancedCheckTime = System.currentTimeMillis(); + cachedAdvancedCheckResult = result; + return result.isDegraded; + } finally { + advancedCheckLock.writeLock().unlock(); + } + } finally { + advancedCheckInProgress.set(false } finally { advancedCheckLock.writeLock().unlock(); } From 824895e52938d65f79269380dea52eba6e0f0b2f Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Wed, 25 Feb 2026 13:47:08 +0530 Subject: [PATCH 19/20] feat(health,version): add index existance, read-only detection, canary write for elasticsearch health check --- .../service/health/HealthService.java | 763 ++++++++++++------ 1 file changed, 503 insertions(+), 260 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java index 6cda78d..7f0366b 100644 --- a/src/main/java/com/iemr/common/identity/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -1,42 +1,46 @@ /* -* AMRIT – Accessible Medical Records via Integrated Technology -* Integrated EHR (Electronic Health Records) Solution -* -* Copyright (C) "Piramal Swasthya Management and Research Institute" -* -* This file is part of AMRIT. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU General Public License as published by -* the Free Software Foundation, either version 3 of the License, or -* (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program. If not, see https://www.gnu.org/licenses/. -*/ + * AMRIT – Accessible Medical Records via Integrated Technology + * Integrated EHR (Electronic Health Records) Solution + * + * Copyright (C) "Piramal Swasthya Management and Research Institute" + * + * This file is part of AMRIT. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ package com.iemr.common.identity.service.health; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.time.Instant; import java.util.LinkedHashMap; import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Supplier; import javax.sql.DataSource; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.zaxxer.hikari.HikariDataSource; import com.zaxxer.hikari.HikariPoolMXBean; import java.lang.management.ManagementFactory; @@ -44,7 +48,9 @@ import javax.management.ObjectName; import jakarta.annotation.PostConstruct; import org.apache.http.HttpHost; +import org.apache.http.client.config.RequestConfig; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,87 +64,106 @@ public class HealthService { private static final Logger logger = LoggerFactory.getLogger(HealthService.class); - private static final String STATUS_KEY = "status"; - private static final String DB_HEALTH_CHECK_QUERY = "SELECT 1 as health_check"; - private static final String STATUS_UP = "UP"; - private static final String STATUS_DOWN = "DOWN"; + + // Status values + private static final String STATUS_KEY = "status"; + private static final String STATUS_UP = "UP"; + private static final String STATUS_DOWN = "DOWN"; private static final String STATUS_DEGRADED = "DEGRADED"; - private static final String ELASTICSEARCH_TYPE = "Elasticsearch"; - private static final int REDIS_TIMEOUT_SECONDS = 3; - - // Severity levels and keys - private static final String SEVERITY_KEY = "severity"; - private static final String SEVERITY_OK = "OK"; - private static final String SEVERITY_WARNING = "WARNING"; + + // Severity values + private static final String SEVERITY_KEY = "severity"; + private static final String SEVERITY_OK = "OK"; + private static final String SEVERITY_WARNING = "WARNING"; private static final String SEVERITY_CRITICAL = "CRITICAL"; - private static final long RESPONSE_TIME_THRESHOLD_MS = 2000; - private static final long ADVANCED_CHECKS_THROTTLE_SECONDS = 30; - - // Response keys - private static final String ERROR_KEY = "error"; - private static final String MESSAGE_KEY = "message"; + + // Response field keys + private static final String ERROR_KEY = "error"; + private static final String MESSAGE_KEY = "message"; private static final String RESPONSE_TIME_KEY = "responseTimeMs"; - - // Component name constants - private static final String MYSQL_COMPONENT = "MySQL"; - private static final String REDIS_COMPONENT = "Redis"; - - // Advanced checks timeout - private static final long ADVANCED_CHECKS_TIMEOUT_MS = 500L; - - // Diagnostic event codes for concise logging - private static final String DIAGNOSTIC_LOCK_WAIT = "MYSQL_LOCK_WAIT"; - private static final String DIAGNOSTIC_SLOW_QUERIES = "MYSQL_SLOW_QUERIES"; + + // Component names + private static final String MYSQL_COMPONENT = "MySQL"; + private static final String REDIS_COMPONENT = "Redis"; + private static final String ELASTICSEARCH_TYPE = "Elasticsearch"; + + // Thresholds + private static final long RESPONSE_TIME_THRESHOLD_MS = 2_000L; + private static final long ADVANCED_CHECKS_THROTTLE_SECONDS = 30L; + private static final long ADVANCED_CHECKS_TIMEOUT_MS = 500L; + + // Diagnostic event codes + private static final String DIAGNOSTIC_LOCK_WAIT = "MYSQL_LOCK_WAIT"; + private static final String DIAGNOSTIC_SLOW_QUERIES = "MYSQL_SLOW_QUERIES"; private static final String DIAGNOSTIC_POOL_EXHAUSTED = "MYSQL_POOL_EXHAUSTED"; - private static final String DIAGNOSTIC_LOG_TEMPLATE = "Diagnostic: {}"; - - private final DataSource dataSource; - private final ExecutorService executorService = Executors.newFixedThreadPool(6); - private final ExecutorService advancedCheckExecutor; - private final RedisTemplate redisTemplate; - private final String elasticsearchHost; - private final int elasticsearchPort; - private final boolean elasticsearchEnabled; + private static final String DIAGNOSTIC_LOG_TEMPLATE = "Diagnostic: {}"; + + // Elasticsearch constants + private static final long ELASTICSEARCH_FUNCTIONAL_CHECKS_THROTTLE_MS = 60_000L; + private static final int ELASTICSEARCH_CONNECT_TIMEOUT_MS = 2_000; + private static final int ELASTICSEARCH_SOCKET_TIMEOUT_MS = 2_000; + private static final int ELASTICSEARCH_CANARY_TIMEOUT_MS = 500; + private static final String ES_CLUSTER_STATUS_YELLOW = "yellow"; + private static final String ES_CLUSTER_STATUS_RED = "red"; + + private static final boolean ADVANCED_HEALTH_CHECKS_ENABLED = true; + + private final DataSource dataSource; + private final ExecutorService advancedCheckExecutor; + private final RedisTemplate redisTemplate; + private final String elasticsearchHost; + private final int elasticsearchPort; + private final boolean elasticsearchEnabled; + private final boolean elasticsearchIndexingRequired; + private final String elasticsearchTargetIndex; + private static final ObjectMapper objectMapper = new ObjectMapper(); + private RestClient elasticsearchRestClient; - private boolean elasticsearchClientReady = false; - - // Advanced checks throttling (thread-safe) - private volatile long lastAdvancedCheckTime = 0; + private boolean elasticsearchClientReady = false; + + private volatile long lastAdvancedCheckTime = 0L; private volatile AdvancedCheckResult cachedAdvancedCheckResult = null; - private final ReentrantReadWriteLock advancedCheckLock = new ReentrantReadWriteLock(); - private final AtomicBoolean advancedCheckInProgress = new AtomicBoolean(false); - - // Advanced checks always enabled - private static final boolean ADVANCED_HEALTH_CHECKS_ENABLED = true; + private final ReentrantReadWriteLock advancedCheckLock = new ReentrantReadWriteLock(); + private final AtomicBoolean advancedCheckInProgress = new AtomicBoolean(false); + + private final AtomicReference elasticsearchCache = new AtomicReference<>(null); + private volatile long lastElasticsearchFunctionalCheckTime = 0L; + private final AtomicBoolean elasticsearchCheckInProgress = new AtomicBoolean(false); + private final AtomicBoolean elasticsearchFunctionalCheckInProgress = new AtomicBoolean(false); + + + public HealthService( + DataSource dataSource, + @Autowired(required = false) RedisTemplate redisTemplate, + @Value("${elasticsearch.host:localhost}") String elasticsearchHost, + @Value("${elasticsearch.port:9200}") int elasticsearchPort, + @Value("${elasticsearch.enabled:false}") boolean elasticsearchEnabled, + @Value("${elasticsearch.target-index:amrit_data}") String elasticsearchTargetIndex, + @Value("${elasticsearch.indexing-required:false}") boolean elasticsearchIndexingRequired) { - public HealthService(DataSource dataSource, - @Autowired(required = false) RedisTemplate redisTemplate, - @Value("${elasticsearch.host:localhost}") String elasticsearchHost, - @Value("${elasticsearch.port:9200}") int elasticsearchPort, - @Value("${elasticsearch.enabled:false}") boolean elasticsearchEnabled) { this.dataSource = dataSource; this.advancedCheckExecutor = Executors.newSingleThreadExecutor(r -> { Thread t = new Thread(r, "health-advanced-check"); t.setDaemon(true); return t; }); - this.redisTemplate = redisTemplate; - this.elasticsearchHost = elasticsearchHost; - this.elasticsearchPort = elasticsearchPort; - this.elasticsearchEnabled = elasticsearchEnabled; + this.redisTemplate = redisTemplate; + this.elasticsearchHost = elasticsearchHost; + this.elasticsearchPort = elasticsearchPort; + this.elasticsearchEnabled = elasticsearchEnabled; + this.elasticsearchIndexingRequired = elasticsearchIndexingRequired; + this.elasticsearchTargetIndex = (elasticsearchTargetIndex != null) ? elasticsearchTargetIndex : "amrit_data"; } - + @PostConstruct public void init() { - // Initialize Elasticsearch RestClient if enabled if (elasticsearchEnabled) { initializeElasticsearchClient(); } } - + @jakarta.annotation.PreDestroy public void cleanup() { - executorService.shutdownNow(); advancedCheckExecutor.shutdownNow(); if (elasticsearchRestClient != null) { try { @@ -148,31 +173,31 @@ public void cleanup() { } } } - + private void initializeElasticsearchClient() { try { this.elasticsearchRestClient = RestClient.builder( - new HttpHost(elasticsearchHost, elasticsearchPort, "http") - ) - .setRequestConfigCallback(requestConfigBuilder -> - requestConfigBuilder - .setConnectTimeout(3000) - .setSocketTimeout(3000) - ) - .build(); + new HttpHost(elasticsearchHost, elasticsearchPort, "http")) + .setRequestConfigCallback(cb -> cb + .setConnectTimeout(ELASTICSEARCH_CONNECT_TIMEOUT_MS) + .setSocketTimeout(ELASTICSEARCH_SOCKET_TIMEOUT_MS)) + .build(); this.elasticsearchClientReady = true; - logger.info("Elasticsearch RestClient initialized for {}:{}", elasticsearchHost, elasticsearchPort); + logger.info("Elasticsearch client initialized (connect/socket timeout: {}ms)", + ELASTICSEARCH_CONNECT_TIMEOUT_MS); } catch (Exception e) { - logger.warn("Failed to initialize Elasticsearch RestClient: {}", e.getMessage()); + logger.warn("Failed to initialize Elasticsearch client: {}", e.getMessage()); this.elasticsearchClientReady = false; } } + public Map checkHealth() { Map healthStatus = new LinkedHashMap<>(); - Map components = new LinkedHashMap<>(); + Map components = new LinkedHashMap<>(); boolean overallHealth = true; + Map mysqlStatus = checkMySQLHealth(); components.put("mysql", mysqlStatus); if (!isHealthy(mysqlStatus)) { @@ -186,11 +211,10 @@ public Map checkHealth() { overallHealth = false; } } - if (elasticsearchEnabled && elasticsearchClientReady) { - Map elasticsearchStatus = checkElasticsearchHealth(); - components.put("elasticsearch", elasticsearchStatus); - if (!isHealthy(elasticsearchStatus)) { + Map esStatus = checkElasticsearchHealth(); + components.put("elasticsearch", esStatus); + if (!isHealthy(esStatus)) { overallHealth = false; } } @@ -198,8 +222,7 @@ public Map checkHealth() { healthStatus.put(STATUS_KEY, overallHealth ? STATUS_UP : STATUS_DOWN); healthStatus.put("timestamp", Instant.now().toString()); healthStatus.put("components", components); - logger.info("Health check completed - Overall status: {}", overallHealth ? STATUS_UP : STATUS_DOWN); - + logger.info("Health check completed – overall: {}", overallHealth ? STATUS_UP : STATUS_DOWN); return healthStatus; } @@ -208,29 +231,25 @@ private Map checkMySQLHealth() { details.put("type", MYSQL_COMPONENT); return performHealthCheck(MYSQL_COMPONENT, details, () -> { - try { - try (Connection connection = dataSource.getConnection()) { - if (connection.isValid(2)) { - try (PreparedStatement stmt = connection.prepareStatement(DB_HEALTH_CHECK_QUERY)) { - stmt.setQueryTimeout(3); - try (ResultSet rs = stmt.executeQuery()) { - if (rs.next() && rs.getInt(1) == 1) { - // Basic check passed - run advanced checks with throttling - boolean isDegraded = performAdvancedMySQLChecksWithThrottle(); - return new HealthCheckResult(true, null, isDegraded); - } - } + try (Connection connection = dataSource.getConnection()) { + if (!connection.isValid(2)) { + return new HealthCheckResult(false, "Connection validation failed", false); + } + try (PreparedStatement stmt = connection.prepareStatement("SELECT 1 as health_check")) { + stmt.setQueryTimeout(3); + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next() && rs.getInt(1) == 1) { + boolean isDegraded = performAdvancedMySQLChecksWithThrottle(); + return new HealthCheckResult(true, null, isDegraded); } } - return new HealthCheckResult(false, "Connection validation failed", false); } + return new HealthCheckResult(false, "Unexpected query result", false); } catch (Exception e) { throw new IllegalStateException(MYSQL_COMPONENT + " connection failed: " + e.getMessage(), e); } }); } - - private Map checkRedisHealth() { Map details = new LinkedHashMap<>(); @@ -238,13 +257,11 @@ private Map checkRedisHealth() { return performHealthCheck(REDIS_COMPONENT, details, () -> { try { - // Run Redis PING synchronously - avoid nested CompletableFuture on same executor - String pong = redisTemplate.execute((RedisCallback) connection -> connection.ping()); - + String pong = redisTemplate.execute((RedisCallback) conn -> conn.ping()); if ("PONG".equals(pong)) { return new HealthCheckResult(true, null, false); } - return new HealthCheckResult(false, "Ping returned unexpected response", false); + return new HealthCheckResult(false, "Unexpected ping response", false); } catch (Exception e) { throw new IllegalStateException("Redis health check failed", e); } @@ -257,44 +274,264 @@ private Map checkElasticsearchHealth() { return performHealthCheck(ELASTICSEARCH_TYPE, details, () -> { if (!elasticsearchClientReady || elasticsearchRestClient == null) { - logger.debug("Elasticsearch RestClient not ready"); - return new HealthCheckResult(false, "Elasticsearch client not ready", false); + return new HealthCheckResult(false, "Service unavailable", false); } - - try { - // Execute a simple cluster health request - Request request = new Request("GET", "/_cluster/health"); - var response = elasticsearchRestClient.performRequest(request); - - int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode == 200) { - logger.debug("{} health check successful", ELASTICSEARCH_TYPE); - return new HealthCheckResult(true, null, false); + + long now = System.currentTimeMillis(); + + // Return fresh cache if still valid + ElasticsearchCacheEntry cached = elasticsearchCache.get(); + if (cached != null && !cached.isExpired(now)) { + logger.debug("Returning cached ES health (age: {}ms, status: {})", + now - cached.timestamp, cached.result.isHealthy ? STATUS_UP : STATUS_DOWN); + return cached.result; + } + + // Single-flight: only one thread probes ES; others use stale cache + if (!elasticsearchCheckInProgress.compareAndSet(false, true)) { + ElasticsearchCacheEntry fallback = elasticsearchCache.get(); + if (fallback != null) { + logger.debug("ES check already in progress – using stale cache"); + return fallback.result; } - return new HealthCheckResult(false, "HTTP " + statusCode, false); - } catch (java.net.ConnectException e) { - logger.error("{} connection refused on {}:{}", ELASTICSEARCH_TYPE, elasticsearchHost, elasticsearchPort, e); - return new HealthCheckResult(false, "Connection refused", false); - } catch (java.io.IOException e) { - logger.error("{} IO error: {}", ELASTICSEARCH_TYPE, e.getMessage(), e); - return new HealthCheckResult(false, "IO Error: " + e.getMessage(), false); + // On cold start with concurrent requests, return DEGRADED (not DOWN) until first result + logger.debug("ES check already in progress with no cache – returning DEGRADED"); + return new HealthCheckResult(true, null, true); + } + + try { + HealthCheckResult result = performElasticsearchHealthCheck(); + elasticsearchCache.set(new ElasticsearchCacheEntry(result, now)); + return result; } catch (Exception e) { - logger.error("{} error: {} - {}", ELASTICSEARCH_TYPE, e.getClass().getSimpleName(), e.getMessage(), e); - return new HealthCheckResult(false, e.getMessage(), false); + logger.debug("Elasticsearch health check exception: {}", e.getClass().getSimpleName()); + HealthCheckResult errorResult = new HealthCheckResult(false, "Service unavailable", false); + elasticsearchCache.set(new ElasticsearchCacheEntry(errorResult, now)); + return errorResult; + } finally { + elasticsearchCheckInProgress.set(false); } }); } + private HealthCheckResult performElasticsearchHealthCheck() throws IOException { + ClusterHealthStatus healthStatus = getClusterHealthStatus(); + if (healthStatus == null) { + // Cluster health unavailable; check if index is reachable to determine degradation vs DOWN + if (indexExists()) { + logger.debug("Cluster health unavailable but index is reachable – returning DEGRADED"); + return new HealthCheckResult(true, null, true); // DEGRADED: index reachable but cluster health offline + } + logger.warn("Cluster health unavailable and index unreachable"); + return new HealthCheckResult(false, "Cluster health unavailable", false); + } + if (ES_CLUSTER_STATUS_RED.equals(healthStatus.status)) { + return new HealthCheckResult(false, "Cluster red", false); + } + + boolean isDegraded = ES_CLUSTER_STATUS_YELLOW.equals(healthStatus.status); + + String functionalCheckError = shouldRunFunctionalChecks() + ? performThrottledFunctionalChecksWithError() + : null; + + if (functionalCheckError != null) { + return new HealthCheckResult(false, functionalCheckError, false); + } + return new HealthCheckResult(true, null, isDegraded); + } + + private ClusterHealthStatus getClusterHealthStatus() { + try { + Request request = new Request("GET", "/_cluster/health"); + applyTimeouts(request, ELASTICSEARCH_CONNECT_TIMEOUT_MS); + var response = elasticsearchRestClient.performRequest(request); + if (response.getStatusLine().getStatusCode() != 200) { + logger.debug("Cluster health returned HTTP {}", response.getStatusLine().getStatusCode()); + return null; + } + String body = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8); + JsonNode root = objectMapper.readTree(body); + String status = root.path("status").asText(); + if (status == null || status.isEmpty()) { + logger.debug("Could not parse cluster status"); + return null; + } + return new ClusterHealthStatus(status); + } catch (java.net.ConnectException | java.net.SocketTimeoutException e) { + logger.debug("Elasticsearch network error: {}", e.getClass().getSimpleName()); + } catch (IOException e) { + logger.debug("Elasticsearch IO error: {}", e.getClass().getSimpleName()); + } catch (Exception e) { + logger.debug("Elasticsearch health check error: {}", e.getClass().getSimpleName()); + } + return null; + } + + private boolean shouldRunFunctionalChecks() { + return (System.currentTimeMillis() - lastElasticsearchFunctionalCheckTime) + >= ELASTICSEARCH_FUNCTIONAL_CHECKS_THROTTLE_MS; + } + + private String performThrottledFunctionalChecksWithError() { + if (!elasticsearchFunctionalCheckInProgress.compareAndSet(false, true)) { + logger.debug("Functional checks already in progress – skipping"); + return null; + } + try { + long now = System.currentTimeMillis(); + + if (!indexExists()) { + logger.warn("Functional check failed: index missing"); + lastElasticsearchFunctionalCheckTime = now; + return "Index missing"; + } + + ReadOnlyCheckResult readOnlyResult = isClusterReadOnly(); + if (readOnlyResult.isReadOnly) { + logger.warn("Functional check failed: cluster is read-only"); + lastElasticsearchFunctionalCheckTime = now; + return "Read-only block"; + } + if (readOnlyResult.isUnableToDetermine) { + logger.warn("Functional check degraded: unable to determine read-only state"); + } + + CanaryWriteResult canaryResult = performCanaryWriteProbe(); + if (!canaryResult.success) { + if (elasticsearchIndexingRequired) { + logger.warn("Functional check failed: canary write unsuccessful – {}", canaryResult.errorCategory); + lastElasticsearchFunctionalCheckTime = now; + return "Canary write failed: " + canaryResult.errorCategory; + } else { + logger.debug("Canary write unsuccessful but indexing not required: {}", canaryResult.errorCategory); + } + } + + lastElasticsearchFunctionalCheckTime = now; + return null; + } finally { + elasticsearchFunctionalCheckInProgress.set(false); + } + } + + private boolean indexExists() { + try { + Request request = new Request("HEAD", "/" + elasticsearchTargetIndex); + applyTimeouts(request, ELASTICSEARCH_CANARY_TIMEOUT_MS); + var response = elasticsearchRestClient.performRequest(request); + return response.getStatusLine().getStatusCode() == 200; + } catch (Exception e) { + logger.debug("Index existence check failed: {}", e.getClass().getSimpleName()); + return false; + } + } + + private ReadOnlyCheckResult isClusterReadOnly() { + try { + Request request = new Request("GET", "/_cluster/settings?include_defaults=true"); + applyTimeouts(request, ELASTICSEARCH_CONNECT_TIMEOUT_MS); + var response = elasticsearchRestClient.performRequest(request); + if (response.getStatusLine().getStatusCode() != 200) { + logger.debug("Cluster settings returned HTTP {}", response.getStatusLine().getStatusCode()); + return new ReadOnlyCheckResult(false, true); + } + String body = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8); + JsonNode root = objectMapper.readTree(body); + boolean hasReadOnlyBlock = hasReadOnlyFlag(root, "read_only"); + boolean hasReadOnlyDeleteBlock = hasReadOnlyFlag(root, "read_only_allow_delete"); + return new ReadOnlyCheckResult(hasReadOnlyBlock || hasReadOnlyDeleteBlock, false); + } catch (java.net.SocketTimeoutException e) { + logger.debug("Read-only check timeout"); + } catch (IOException e) { + logger.debug("Read-only check IO error: {}", e.getClass().getSimpleName()); + } catch (Exception e) { + logger.debug("Read-only check failed: {}", e.getClass().getSimpleName()); + } + return new ReadOnlyCheckResult(false, true); + } + + private boolean hasReadOnlyFlag(JsonNode root, String flag) { + String[] paths = { + "/persistent/cluster/blocks/" + flag, + "/transient/cluster/blocks/" + flag, + "/defaults/cluster/blocks/" + flag + }; + for (String path : paths) { + try { + JsonNode node = root.at(path); + if (node != null && !node.isMissingNode()) { + if ((node.isBoolean() && node.asBoolean()) || + (node.isTextual() && "true".equalsIgnoreCase(node.asText()))) { + logger.debug("Found read-only flag at {}", path); + return true; + } + } + } catch (Exception e) { + logger.debug("Error checking JSON pointer {}: {}", path, e.getClass().getSimpleName()); + } + } + return false; + } + + private void applyTimeouts(Request request, int timeoutMs) { + RequestOptions options = RequestOptions.DEFAULT.toBuilder() + .setRequestConfig(RequestConfig.custom() + .setConnectTimeout(timeoutMs) + .setSocketTimeout(timeoutMs) + .build()) + .build(); + request.setOptions(options); + } + + private CanaryWriteResult performCanaryWriteProbe() { + String canaryDocId = "health-check-canary"; + try { + String canaryBody = "{\"probe\":true,\"timestamp\":\"" + Instant.now() + "\"}"; + + // FIX: Use PUT (not POST) for a document with a specific ID + Request writeRequest = new Request("PUT", "/" + elasticsearchTargetIndex + "/_doc/" + canaryDocId); + applyTimeouts(writeRequest, ELASTICSEARCH_CANARY_TIMEOUT_MS); + writeRequest.setEntity(new org.apache.http.entity.StringEntity(canaryBody, StandardCharsets.UTF_8)); + writeRequest.addParameter("refresh", "true"); + + var writeResponse = elasticsearchRestClient.performRequest(writeRequest); + if (writeResponse.getStatusLine().getStatusCode() > 299) { + logger.debug("Canary write failed with HTTP {}", writeResponse.getStatusLine().getStatusCode()); + return new CanaryWriteResult(false, "Write rejected"); + } + + // Best-effort delete + try { + Request deleteRequest = new Request("DELETE", "/" + elasticsearchTargetIndex + "/_doc/" + canaryDocId); + applyTimeouts(deleteRequest, ELASTICSEARCH_CANARY_TIMEOUT_MS); + elasticsearchRestClient.performRequest(deleteRequest); + } catch (Exception e) { + logger.debug("Canary delete warning: {}", e.getClass().getSimpleName()); + } + + return new CanaryWriteResult(true, null); + } catch (java.net.SocketTimeoutException e) { + logger.debug("Canary probe timeout"); + return new CanaryWriteResult(false, "Timeout"); + } catch (java.net.ConnectException e) { + logger.debug("Canary probe connection refused"); + return new CanaryWriteResult(false, "Connection refused"); + } catch (Exception e) { + logger.debug("Canary probe failed: {}", e.getClass().getSimpleName()); + return new CanaryWriteResult(false, "Write failed"); + } + } + + private Map performHealthCheck(String componentName, Map details, Supplier checker) { Map status = new LinkedHashMap<>(); long startTime = System.currentTimeMillis(); - try { - HealthCheckResult result = checker.get(); - long responseTime = System.currentTimeMillis() - startTime; - + HealthCheckResult result = checker.get(); + long responseTime = System.currentTimeMillis() - startTime; details.put(RESPONSE_TIME_KEY, responseTime); if (result.isHealthy) { @@ -302,10 +539,8 @@ private Map performHealthCheck(String componentName, } else { buildUnhealthyStatus(status, details, componentName, result); } - status.put("details", details); return status; - } catch (Exception e) { long responseTime = System.currentTimeMillis() - startTime; return buildExceptionStatus(status, details, componentName, e, responseTime); @@ -313,84 +548,79 @@ private Map performHealthCheck(String componentName, } private void buildHealthyStatus(Map status, - String componentName, long responseTime, HealthCheckResult result) { - logger.debug("{} health check: UP ({}ms)", componentName, responseTime); - - // Determine status based on health, response time, and degradation flags - String statusValue = result.isDegraded ? STATUS_DEGRADED : STATUS_UP; - status.put(STATUS_KEY, statusValue); - - String severity = determineSeverity(result.isHealthy, responseTime, result.isDegraded); - status.put(SEVERITY_KEY, severity); - - // Include message if there's an error (informational when healthy) + String componentName, long responseTime, HealthCheckResult result) { + logger.debug("{} health check: {} ({}ms)", + componentName, result.isDegraded ? STATUS_DEGRADED : STATUS_UP, responseTime); + status.put(STATUS_KEY, result.isDegraded ? STATUS_DEGRADED : STATUS_UP); + status.put(SEVERITY_KEY, determineSeverity(true, responseTime, result.isDegraded)); if (result.error != null) { status.put(MESSAGE_KEY, result.error); } } private void buildUnhealthyStatus(Map status, Map details, - String componentName, HealthCheckResult result) { - String safeError = result.error != null ? result.error : "Health check failed"; - logger.warn("{} health check failed: {}", componentName, safeError); - status.put(STATUS_KEY, STATUS_DOWN); + String componentName, HealthCheckResult result) { + String internalError = (result.error != null) ? result.error : "Health check failed"; + logger.warn("{} health check failed: {}", componentName, internalError); + status.put(STATUS_KEY, STATUS_DOWN); status.put(SEVERITY_KEY, SEVERITY_CRITICAL); - details.put(ERROR_KEY, safeError); - details.put("errorType", "CheckFailed"); + // Sanitized outward message – no topology leakage + details.put(ERROR_KEY, "Dependency unavailable"); + // For Elasticsearch, sanitize detailed failure reasons; keep real reason in logs only + String exposedCategory = ELASTICSEARCH_TYPE.equals(componentName) + ? "DEPENDENCY_FAILURE" + : internalError; + details.put("errorCategory", exposedCategory); + details.put("errorType", "CheckFailed"); } private Map buildExceptionStatus(Map status, Map details, - String componentName, Exception e, long responseTime) { - logger.error("{} health check failed with exception: {}", componentName, e.getMessage(), e); - - String errorMessage = e.getCause() != null ? e.getCause().getMessage() : e.getMessage(); - - status.put(STATUS_KEY, STATUS_DOWN); + String componentName, Exception e, long responseTime) { + logger.error("{} health check threw exception: {}", componentName, e.getMessage(), e); + status.put(STATUS_KEY, STATUS_DOWN); status.put(SEVERITY_KEY, SEVERITY_CRITICAL); details.put(RESPONSE_TIME_KEY, responseTime); - details.put(ERROR_KEY, errorMessage != null ? errorMessage : "Health check failed"); + // FIX: Sanitize error message – do not expose raw exception detail to consumers + details.put(ERROR_KEY, "Dependency unavailable"); + details.put("errorCategory", "CheckException"); + details.put("errorType", "Exception"); status.put("details", details); - return status; } + private String determineSeverity(boolean isHealthy, long responseTimeMs, boolean isDegraded) { - if (!isHealthy) { - return SEVERITY_CRITICAL; - } - - if (isDegraded) { - return SEVERITY_WARNING; - } - - if (responseTimeMs > RESPONSE_TIME_THRESHOLD_MS) { - return SEVERITY_WARNING; - } - + if (!isHealthy) return SEVERITY_CRITICAL; + if (isDegraded) return SEVERITY_WARNING; + if (responseTimeMs > RESPONSE_TIME_THRESHOLD_MS) return SEVERITY_WARNING; return SEVERITY_OK; } private boolean isHealthy(Map componentStatus) { - return STATUS_UP.equals(componentStatus.get(STATUS_KEY)); + Object s = componentStatus.get(STATUS_KEY); + return STATUS_UP.equals(s) || STATUS_DEGRADED.equals(s); } - private boolean performAdvancedMySQLChecksWithThrottle() { if (!ADVANCED_HEALTH_CHECKS_ENABLED) { - return false; // Advanced checks disabled + return false; } - + long currentTime = System.currentTimeMillis(); - + + // --- Phase 1: try to serve from cache (read lock) --- advancedCheckLock.readLock().lock(); try { - if (cachedAdvancedCheckResult != null && - (currentTime - lastAdvancedCheckTime) < ADVANCED_CHECKS_THROTTLE_SECONDS * 1000) { + if (cachedAdvancedCheckResult != null && + (currentTime - lastAdvancedCheckTime) < ADVANCED_CHECKS_THROTTLE_SECONDS * 1_000L) { return cachedAdvancedCheckResult.isDegraded; } } finally { advancedCheckLock.readLock().unlock(); - // Only one thread may submit; others fall back to the (stale) cache + } + + // --- Phase 2: single-flight guard --- if (!advancedCheckInProgress.compareAndSet(false, true)) { + // Another thread is refreshing – return stale cache (safe fallback) advancedCheckLock.readLock().lock(); try { return cachedAdvancedCheckResult != null && cachedAdvancedCheckResult.isDegraded; @@ -398,84 +628,72 @@ private boolean performAdvancedMySQLChecksWithThrottle() { advancedCheckLock.readLock().unlock(); } } - + + // --- Phase 3: run DB checks outside any lock --- try { - // DB I/O outside the lock to prevent lock contention AdvancedCheckResult result = performAdvancedMySQLChecks(); - - // Re-acquire write lock only for atomic cache update + + // --- Phase 4: write-lock for atomic cache update --- advancedCheckLock.writeLock().lock(); try { - lastAdvancedCheckTime = System.currentTimeMillis(); + lastAdvancedCheckTime = System.currentTimeMillis(); cachedAdvancedCheckResult = result; return result.isDegraded; } finally { advancedCheckLock.writeLock().unlock(); } } finally { - advancedCheckInProgress.set(false - } finally { - advancedCheckLock.writeLock().unlock(); + advancedCheckInProgress.set(false); } } private AdvancedCheckResult performAdvancedMySQLChecks() { - try { - try (Connection connection = dataSource.getConnection()) { - return executeAdvancedCheckAsync(connection); - } + try (Connection connection = dataSource.getConnection()) { + return executeAdvancedCheckAsync(connection); } catch (Exception e) { logger.debug("Failed to get connection for advanced checks: {}", e.getMessage()); return new AdvancedCheckResult(true); } } - + private AdvancedCheckResult executeAdvancedCheckAsync(Connection connection) { + Future future = advancedCheckExecutor.submit( + () -> performAdvancedCheckLogic(connection)); try { - Future future = - advancedCheckExecutor.submit( - () -> performAdvancedCheckLogic(connection) - ); - return future.get(ADVANCED_CHECKS_TIMEOUT_MS, java.util.concurrent.TimeUnit.MILLISECONDS); } catch (java.util.concurrent.TimeoutException e) { - logger.debug("Advanced checks timeout, marking degraded"); - return new AdvancedCheckResult(true); + logger.debug("Advanced checks timed out – marking degraded"); + future.cancel(true); } catch (ExecutionException e) { if (e.getCause() instanceof InterruptedException) { Thread.currentThread().interrupt(); } - logger.debug("Advanced checks execution failed, marking degraded"); - return new AdvancedCheckResult(true); + logger.debug("Advanced checks execution failed – marking degraded"); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - logger.debug("Advanced checks interrupted, marking degraded"); - return new AdvancedCheckResult(true); + logger.debug("Advanced checks interrupted – marking degraded"); } catch (Exception e) { - logger.debug("Advanced checks encountered exception, marking degraded"); - return new AdvancedCheckResult(true); + logger.debug("Advanced checks encountered exception – marking degraded"); } + return new AdvancedCheckResult(true); } - + private AdvancedCheckResult performAdvancedCheckLogic(Connection connection) { try { boolean hasIssues = false; - + if (hasLockWaits(connection)) { logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_LOCK_WAIT); hasIssues = true; } - if (hasSlowQueries(connection)) { logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_SLOW_QUERIES); hasIssues = true; } - if (hasConnectionPoolExhaustion()) { logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_POOL_EXHAUSTED); hasIssues = true; } - return new AdvancedCheckResult(hasIssues); } catch (Exception e) { logger.debug("Advanced check logic encountered exception"); @@ -484,17 +702,17 @@ private AdvancedCheckResult performAdvancedCheckLogic(Connection connection) { } private boolean hasLockWaits(Connection connection) { - try (PreparedStatement stmt = connection.prepareStatement( - "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + - "WHERE (state = 'Waiting for table metadata lock' " + - " OR state = 'Waiting for row lock' " + - " OR state = 'Waiting for lock') " + - "AND user = USER()")) { + String sql = + "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + + "WHERE (state = 'Waiting for table metadata lock' " + + " OR state = 'Waiting for row lock' " + + " OR state = 'Waiting for lock') " + + "AND user = USER()"; + try (PreparedStatement stmt = connection.prepareStatement(sql)) { stmt.setQueryTimeout(2); try (ResultSet rs = stmt.executeQuery()) { if (rs.next()) { - int lockCount = rs.getInt(1); - return lockCount > 0; + return rs.getInt(1) > 0; } } } catch (Exception e) { @@ -502,16 +720,17 @@ private boolean hasLockWaits(Connection connection) { } return false; } + private boolean hasSlowQueries(Connection connection) { - try (PreparedStatement stmt = connection.prepareStatement( - "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + - "WHERE command != 'Sleep' AND time > ? AND user = SUBSTRING_INDEX(USER(), '@', 1)")) { + String sql = + "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + + "WHERE command != 'Sleep' AND time > ? AND user = SUBSTRING_INDEX(USER(), '@', 1)"; + try (PreparedStatement stmt = connection.prepareStatement(sql)) { stmt.setQueryTimeout(2); - stmt.setInt(1, 10); // Queries running longer than 10 seconds + stmt.setInt(1, 10); // queries running > 10 seconds try (ResultSet rs = stmt.executeQuery()) { if (rs.next()) { - int slowQueryCount = rs.getInt(1); - return slowQueryCount > 3; // Alert if more than 3 slow queries + return rs.getInt(1) > 3; // more than 3 slow queries } } } catch (Exception e) { @@ -521,34 +740,27 @@ private boolean hasSlowQueries(Connection connection) { } private boolean hasConnectionPoolExhaustion() { - // Use HikariCP metrics if available if (dataSource instanceof HikariDataSource hikariDataSource) { try { HikariPoolMXBean poolMXBean = hikariDataSource.getHikariPoolMXBean(); - if (poolMXBean != null) { int activeConnections = poolMXBean.getActiveConnections(); - int maxPoolSize = hikariDataSource.getMaximumPoolSize(); - - // Alert if > 80% of pool is exhausted - int threshold = (int) (maxPoolSize * 0.8); + int maxPoolSize = hikariDataSource.getMaximumPoolSize(); + int threshold = (int) (maxPoolSize * 0.8); return activeConnections > threshold; } } catch (Exception e) { - logger.debug("Could not retrieve HikariCP pool metrics"); + logger.debug("Could not retrieve HikariCP pool metrics directly"); } } - - // Fallback: try to get pool metrics via JMX if HikariCP is not directly available return checkPoolMetricsViaJMX(); } private boolean checkPoolMetricsViaJMX() { try { - MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer(); - ObjectName objectName = new ObjectName("com.zaxxer.hikari:type=Pool (*)"); - var mBeans = mBeanServer.queryMBeans(objectName, null); - + MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer(); + ObjectName objectName = new ObjectName("com.zaxxer.hikari:type=Pool (*)"); + var mBeans = mBeanServer.queryMBeans(objectName, null); for (var mBean : mBeans) { if (evaluatePoolMetrics(mBeanServer, mBean.getObjectName())) { return true; @@ -557,8 +769,6 @@ private boolean checkPoolMetricsViaJMX() { } catch (Exception e) { logger.debug("Could not access HikariCP pool metrics via JMX"); } - - // No pool metrics available - disable this check logger.debug("Pool exhaustion check disabled: HikariCP metrics unavailable"); return false; } @@ -566,8 +776,7 @@ private boolean checkPoolMetricsViaJMX() { private boolean evaluatePoolMetrics(MBeanServer mBeanServer, ObjectName objectName) { try { Integer activeConnections = (Integer) mBeanServer.getAttribute(objectName, "ActiveConnections"); - Integer maximumPoolSize = (Integer) mBeanServer.getAttribute(objectName, "MaximumPoolSize"); - + Integer maximumPoolSize = (Integer) mBeanServer.getAttribute(objectName, "MaximumPoolSize"); if (activeConnections != null && maximumPoolSize != null) { int threshold = (int) (maximumPoolSize * 0.8); return activeConnections > threshold; @@ -578,23 +787,57 @@ private boolean evaluatePoolMetrics(MBeanServer mBeanServer, ObjectName objectNa return false; } + private static class AdvancedCheckResult { final boolean isDegraded; - - AdvancedCheckResult(boolean isDegraded) { - this.isDegraded = isDegraded; - } + AdvancedCheckResult(boolean isDegraded) { this.isDegraded = isDegraded; } } private static class HealthCheckResult { final boolean isHealthy; - final String error; + final String error; final boolean isDegraded; - HealthCheckResult(boolean isHealthy, String error, boolean isDegraded) { - this.isHealthy = isHealthy; - this.error = error; + this.isHealthy = isHealthy; + this.error = error; this.isDegraded = isDegraded; } } -} + + private static class ElasticsearchCacheEntry { + final HealthCheckResult result; + final long timestamp; + ElasticsearchCacheEntry(HealthCheckResult result, long timestamp) { + this.result = result; + this.timestamp = timestamp; + } + /** UP results cache for 30 s; DOWN results for 5 s for faster recovery. */ + boolean isExpired(long now) { + long ttlMs = result.isHealthy ? 30_000L : 5_000L; + return (now - timestamp) >= ttlMs; + } + } + + private static class ClusterHealthStatus { + final String status; + ClusterHealthStatus(String status) { this.status = status; } + } + + private static class ReadOnlyCheckResult { + final boolean isReadOnly; + final boolean isUnableToDetermine; + ReadOnlyCheckResult(boolean isReadOnly, boolean isUnableToDetermine) { + this.isReadOnly = isReadOnly; + this.isUnableToDetermine = isUnableToDetermine; + } + } + + private static class CanaryWriteResult { + final boolean success; + final String errorCategory; + CanaryWriteResult(boolean success, String errorCategory) { + this.success = success; + this.errorCategory = errorCategory; + } + } +} \ No newline at end of file From e9c7c336055748ca60984ddadfc19d0319a3b4c2 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Wed, 25 Feb 2026 13:54:40 +0530 Subject: [PATCH 20/20] refactor(health): reduce cognitive complexity, remove dead throws, and clean code smells --- .../service/health/HealthService.java | 116 ++++++++++-------- 1 file changed, 64 insertions(+), 52 deletions(-) diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java index 7f0366b..f233d72 100644 --- a/src/main/java/com/iemr/common/identity/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -271,50 +271,61 @@ private Map checkRedisHealth() { private Map checkElasticsearchHealth() { Map details = new LinkedHashMap<>(); details.put("type", ELASTICSEARCH_TYPE); + return performHealthCheck(ELASTICSEARCH_TYPE, details, this::getElasticsearchHealthResult); + } - return performHealthCheck(ELASTICSEARCH_TYPE, details, () -> { - if (!elasticsearchClientReady || elasticsearchRestClient == null) { - return new HealthCheckResult(false, "Service unavailable", false); - } + private HealthCheckResult getElasticsearchHealthResult() { + if (!elasticsearchClientReady || elasticsearchRestClient == null) { + return new HealthCheckResult(false, "Service unavailable", false); + } - long now = System.currentTimeMillis(); + long now = System.currentTimeMillis(); + HealthCheckResult cached = getCachedElasticsearchHealth(now); + if (cached != null) { + return cached; + } - // Return fresh cache if still valid - ElasticsearchCacheEntry cached = elasticsearchCache.get(); - if (cached != null && !cached.isExpired(now)) { - logger.debug("Returning cached ES health (age: {}ms, status: {})", - now - cached.timestamp, cached.result.isHealthy ? STATUS_UP : STATUS_DOWN); - return cached.result; - } + return performElasticsearchHealthCheckWithCache(now); + } - // Single-flight: only one thread probes ES; others use stale cache - if (!elasticsearchCheckInProgress.compareAndSet(false, true)) { - ElasticsearchCacheEntry fallback = elasticsearchCache.get(); - if (fallback != null) { - logger.debug("ES check already in progress – using stale cache"); - return fallback.result; - } - // On cold start with concurrent requests, return DEGRADED (not DOWN) until first result - logger.debug("ES check already in progress with no cache – returning DEGRADED"); - return new HealthCheckResult(true, null, true); - } + private HealthCheckResult getCachedElasticsearchHealth(long now) { + ElasticsearchCacheEntry cached = elasticsearchCache.get(); + if (cached != null && !cached.isExpired(now)) { + logger.debug("Returning cached ES health (age: {}ms, status: {})", + now - cached.timestamp, cached.result.isHealthy ? STATUS_UP : STATUS_DOWN); + return cached.result; + } + return null; + } - try { - HealthCheckResult result = performElasticsearchHealthCheck(); - elasticsearchCache.set(new ElasticsearchCacheEntry(result, now)); - return result; - } catch (Exception e) { - logger.debug("Elasticsearch health check exception: {}", e.getClass().getSimpleName()); - HealthCheckResult errorResult = new HealthCheckResult(false, "Service unavailable", false); - elasticsearchCache.set(new ElasticsearchCacheEntry(errorResult, now)); - return errorResult; - } finally { - elasticsearchCheckInProgress.set(false); + private HealthCheckResult performElasticsearchHealthCheckWithCache(long now) { + // Single-flight: only one thread probes ES; others use stale cache + if (!elasticsearchCheckInProgress.compareAndSet(false, true)) { + ElasticsearchCacheEntry fallback = elasticsearchCache.get(); + if (fallback != null) { + logger.debug("ES check already in progress – using stale cache"); + return fallback.result; } - }); + // On cold start with concurrent requests, return DEGRADED (not DOWN) until first result + logger.debug("ES check already in progress with no cache – returning DEGRADED"); + return new HealthCheckResult(true, null, true); + } + + try { + HealthCheckResult result = performElasticsearchHealthCheck(); + elasticsearchCache.set(new ElasticsearchCacheEntry(result, now)); + return result; + } catch (Exception e) { + logger.debug("Elasticsearch health check exception: {}", e.getClass().getSimpleName()); + HealthCheckResult errorResult = new HealthCheckResult(false, "Service unavailable", false); + elasticsearchCache.set(new ElasticsearchCacheEntry(errorResult, now)); + return errorResult; + } finally { + elasticsearchCheckInProgress.set(false); + } } - private HealthCheckResult performElasticsearchHealthCheck() throws IOException { + private HealthCheckResult performElasticsearchHealthCheck() { ClusterHealthStatus healthStatus = getClusterHealthStatus(); if (healthStatus == null) { // Cluster health unavailable; check if index is reachable to determine degradation vs DOWN @@ -352,7 +363,7 @@ private ClusterHealthStatus getClusterHealthStatus() { } String body = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8); JsonNode root = objectMapper.readTree(body); - String status = root.path("status").asText(); + String status = root.path(STATUS_KEY).asText(); if (status == null || status.isEmpty()) { logger.debug("Could not parse cluster status"); return null; @@ -460,12 +471,11 @@ private boolean hasReadOnlyFlag(JsonNode root, String flag) { for (String path : paths) { try { JsonNode node = root.at(path); - if (node != null && !node.isMissingNode()) { - if ((node.isBoolean() && node.asBoolean()) || - (node.isTextual() && "true".equalsIgnoreCase(node.asText()))) { - logger.debug("Found read-only flag at {}", path); - return true; - } + if (node != null && !node.isMissingNode() && + ((node.isBoolean() && node.asBoolean()) || + (node.isTextual() && "true".equalsIgnoreCase(node.asText())))) { + logger.debug("Found read-only flag at {}", path); + return true; } } catch (Exception e) { logger.debug("Error checking JSON pointer {}: {}", path, e.getClass().getSimpleName()); @@ -484,6 +494,16 @@ private void applyTimeouts(Request request, int timeoutMs) { request.setOptions(options); } + private void performCanaryDelete(String canaryDocId) { + try { + Request deleteRequest = new Request("DELETE", "/" + elasticsearchTargetIndex + "/_doc/" + canaryDocId); + applyTimeouts(deleteRequest, ELASTICSEARCH_CANARY_TIMEOUT_MS); + elasticsearchRestClient.performRequest(deleteRequest); + } catch (Exception e) { + logger.debug("Canary delete warning: {}", e.getClass().getSimpleName()); + } + } + private CanaryWriteResult performCanaryWriteProbe() { String canaryDocId = "health-check-canary"; try { @@ -501,15 +521,7 @@ private CanaryWriteResult performCanaryWriteProbe() { return new CanaryWriteResult(false, "Write rejected"); } - // Best-effort delete - try { - Request deleteRequest = new Request("DELETE", "/" + elasticsearchTargetIndex + "/_doc/" + canaryDocId); - applyTimeouts(deleteRequest, ELASTICSEARCH_CANARY_TIMEOUT_MS); - elasticsearchRestClient.performRequest(deleteRequest); - } catch (Exception e) { - logger.debug("Canary delete warning: {}", e.getClass().getSimpleName()); - } - + performCanaryDelete(canaryDocId); return new CanaryWriteResult(true, null); } catch (java.net.SocketTimeoutException e) { logger.debug("Canary probe timeout");