From 39f108e49613936baec153a2924b1263c0475afe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9=20=D0=A1=D0=BE?= =?UTF-8?q?=D0=BB=D0=BE=D0=B2=D1=8C=D0=B5=D0=B2?= Date: Sat, 30 May 2026 21:04:57 +0300 Subject: [PATCH] Handle Ordinis outage in TGU service Classify Ordinis classifier transport failures and empty responses as unavailable, keep stale platform cache when possible, and expose Prometheus metrics/alerting for the outage signal. --- config-repo/pcp-tgu-service.yaml | 2 +- .../templates/prometheusrule.yaml | 21 +++++++ helm/pcp-tgu-service/values.yaml | 6 ++ services/pcp-tgu-service/build.gradle.kts | 1 + .../api/PlatformsClassifierClient.kt | 37 ++++++++++-- .../service/OrdinisAvailabilityMetrics.kt | 40 +++++++++++++ .../service/PlatformService.kt | 43 +++++++++++++- .../src/main/resources/application.yml | 6 ++ .../api/ExternalPointsClientTest.kt | 6 +- .../api/PlatformsClassifierClientTest.kt | 31 ++++++++-- .../service/PlatformServiceTest.kt | 58 ++++++++++++++++++- 11 files changed, 235 insertions(+), 16 deletions(-) create mode 100644 helm/pcp-tgu-service/templates/prometheusrule.yaml create mode 100644 services/pcp-tgu-service/src/main/kotlin/space/nstart/pcp_tgu_service/service/OrdinisAvailabilityMetrics.kt diff --git a/config-repo/pcp-tgu-service.yaml b/config-repo/pcp-tgu-service.yaml index 221d943..5dd0848 100644 --- a/config-repo/pcp-tgu-service.yaml +++ b/config-repo/pcp-tgu-service.yaml @@ -23,7 +23,7 @@ management: endpoints: web: exposure: - include: health,info,metrics + include: health,info,metrics,prometheus springdoc: swagger-ui: diff --git a/helm/pcp-tgu-service/templates/prometheusrule.yaml b/helm/pcp-tgu-service/templates/prometheusrule.yaml new file mode 100644 index 0000000..2cbda7b --- /dev/null +++ b/helm/pcp-tgu-service/templates/prometheusrule.yaml @@ -0,0 +1,21 @@ +{{- if .Values.prometheusRule.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "pcp-tgu-service.fullname" . }}-alerts + labels: + {{- include "pcp-tgu-service.labels" . | nindent 4 }} +spec: + groups: + - name: {{ include "pcp-tgu-service.name" . }}.ordinis + rules: + - alert: PcpTguOrdinisUnavailable + expr: pcp_tgu_ordinis_unavailable == 1 + for: {{ index .Values.prometheusRule.ordinisUnavailable "for" | quote }} + labels: + severity: {{ .Values.prometheusRule.ordinisUnavailable.severity | quote }} + service: {{ include "pcp-tgu-service.name" . | quote }} + annotations: + summary: "Ordinis classifier is unavailable for pcp-tgu-service" + description: "The last pcp-tgu-service attempt to fetch platforms from Ordinis failed. Existing stale platform cache is kept when available." +{{- end }} diff --git a/helm/pcp-tgu-service/values.yaml b/helm/pcp-tgu-service/values.yaml index 457e6f0..7730b62 100644 --- a/helm/pcp-tgu-service/values.yaml +++ b/helm/pcp-tgu-service/values.yaml @@ -35,5 +35,11 @@ resources: {} podAnnotations: {} podLabels: {} +prometheusRule: + enabled: false + ordinisUnavailable: + for: 5m + severity: warning + nameOverride: "" fullnameOverride: "" diff --git a/services/pcp-tgu-service/build.gradle.kts b/services/pcp-tgu-service/build.gradle.kts index 15f6bff..05636e5 100644 --- a/services/pcp-tgu-service/build.gradle.kts +++ b/services/pcp-tgu-service/build.gradle.kts @@ -20,6 +20,7 @@ dependencies { // Spring Boot implementation("org.springframework.boot:spring-boot-starter-webflux") implementation("org.springframework.boot:spring-boot-starter-actuator") + runtimeOnly("io.micrometer:micrometer-registry-prometheus") implementation("org.springdoc:springdoc-openapi-starter-webflux-ui:3.0.0") implementation("org.springframework.boot:spring-boot-starter-data-jpa") implementation("org.springframework.boot:spring-boot-starter-flyway") diff --git a/services/pcp-tgu-service/src/main/kotlin/space/nstart/pcp_tgu_service/integration/api/PlatformsClassifierClient.kt b/services/pcp-tgu-service/src/main/kotlin/space/nstart/pcp_tgu_service/integration/api/PlatformsClassifierClient.kt index 8377293..5bffba8 100644 --- a/services/pcp-tgu-service/src/main/kotlin/space/nstart/pcp_tgu_service/integration/api/PlatformsClassifierClient.kt +++ b/services/pcp-tgu-service/src/main/kotlin/space/nstart/pcp_tgu_service/integration/api/PlatformsClassifierClient.kt @@ -5,6 +5,8 @@ import space.nstart.pcp_tgu_service.config.ClassifierProperties import space.nstart.pcp_tgu_service.dto.NsiPlatformDto import org.springframework.stereotype.Component import org.springframework.web.reactive.function.client.WebClient +import org.springframework.web.reactive.function.client.WebClientException +import org.springframework.web.reactive.function.client.WebClientResponseException /** * HTTP-клиент для получения списка космических аппаратов из классификатора. @@ -22,10 +24,33 @@ class PlatformsClassifierClient( * Загружает список платформ из классификатора НСИ. */ fun fetchPlatforms(): List = - webClient.get() - .retrieve() - .bodyToMono(Array::class.java) - .map { platforms -> platforms.toList() } - .block() - ?: throw IllegalStateException("Classifier response body is empty: ${classifierProperties.platformsUrl}") + try { + val platforms = webClient.get() + .retrieve() + .bodyToMono(Array::class.java) + .block() + + platforms?.toList() + ?: throw OrdinisUnavailableException( + classifierUrl = classifierProperties.platformsUrl, + statusCode = null, + cause = IllegalStateException("Classifier response body is empty: ${classifierProperties.platformsUrl}") + ) + } catch (ex: WebClientException) { + throw OrdinisUnavailableException( + classifierUrl = classifierProperties.platformsUrl, + statusCode = (ex as? WebClientResponseException)?.statusCode?.value(), + cause = ex + ) + } } + +/** + * Семантическая ошибка недоступности Ordinis/НСИ, чтобы service layer мог безопасно + * отличить outage внешней зависимости от ошибок маппинга или доменной логики. + */ +class OrdinisUnavailableException( + val classifierUrl: String, + val statusCode: Int?, + cause: Throwable +) : RuntimeException("Ordinis classifier is unavailable: $classifierUrl", cause) diff --git a/services/pcp-tgu-service/src/main/kotlin/space/nstart/pcp_tgu_service/service/OrdinisAvailabilityMetrics.kt b/services/pcp-tgu-service/src/main/kotlin/space/nstart/pcp_tgu_service/service/OrdinisAvailabilityMetrics.kt new file mode 100644 index 0000000..02911c1 --- /dev/null +++ b/services/pcp-tgu-service/src/main/kotlin/space/nstart/pcp_tgu_service/service/OrdinisAvailabilityMetrics.kt @@ -0,0 +1,40 @@ +package space.nstart.pcp_tgu_service.service + +import io.micrometer.core.instrument.Counter +import io.micrometer.core.instrument.Gauge +import io.micrometer.core.instrument.MeterRegistry +import org.springframework.stereotype.Component +import java.util.concurrent.atomic.AtomicInteger + +/** + * Метрики доступности Ordinis/НСИ для alerting по последней попытке синхронизации. + */ +@Component +class OrdinisAvailabilityMetrics( + meterRegistry: MeterRegistry +) { + private val unavailableGaugeValue = AtomicInteger(0) + private val unavailableCounter = Counter.builder(UNAVAILABLE_TOTAL) + .description("Total Ordinis classifier fetch failures detected by pcp-tgu-service") + .register(meterRegistry) + + init { + Gauge.builder(UNAVAILABLE, unavailableGaugeValue) { value -> value.get().toDouble() } + .description("1 when the last Ordinis classifier fetch failed, 0 after a successful fetch") + .register(meterRegistry) + } + + fun markAvailable() { + unavailableGaugeValue.set(0) + } + + fun markUnavailable() { + unavailableGaugeValue.set(1) + unavailableCounter.increment() + } + + companion object { + const val UNAVAILABLE = "pcp_tgu_ordinis_unavailable" + const val UNAVAILABLE_TOTAL = "pcp_tgu_ordinis_unavailable_total" + } +} diff --git a/services/pcp-tgu-service/src/main/kotlin/space/nstart/pcp_tgu_service/service/PlatformService.kt b/services/pcp-tgu-service/src/main/kotlin/space/nstart/pcp_tgu_service/service/PlatformService.kt index 01f43b3..96fc185 100644 --- a/services/pcp-tgu-service/src/main/kotlin/space/nstart/pcp_tgu_service/service/PlatformService.kt +++ b/services/pcp-tgu-service/src/main/kotlin/space/nstart/pcp_tgu_service/service/PlatformService.kt @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import space.nstart.pcp_tgu_service.config.ClassifierProperties import space.nstart.pcp_tgu_service.domain.PlatformCatalogItem import space.nstart.pcp_tgu_service.dto.NsiPlatformDto +import space.nstart.pcp_tgu_service.integration.api.OrdinisUnavailableException import space.nstart.pcp_tgu_service.integration.api.PlatformsClassifierClient import org.slf4j.LoggerFactory import org.springframework.stereotype.Service @@ -16,6 +17,7 @@ class PlatformService( private val platformsClassifierClient: PlatformsClassifierClient, private val classifierProperties: ClassifierProperties, private val objectMapper: ObjectMapper, + private val ordinisAvailabilityMetrics: OrdinisAvailabilityMetrics, private val clock: Clock = Clock.systemUTC() ) { private val log = LoggerFactory.getLogger(javaClass) @@ -63,7 +65,40 @@ class PlatformService( } private fun fetchAndCachePlatforms(now: Instant): List { - return cachePlatforms(platformsClassifierClient.fetchPlatforms(), now) + return try { + val platforms = cachePlatforms(platformsClassifierClient.fetchPlatforms(), now) + ordinisAvailabilityMetrics.markAvailable() + platforms + } catch (ex: OrdinisUnavailableException) { + ordinisAvailabilityMetrics.markUnavailable() + useStaleCacheOrRethrow(ex) + } + } + + private fun useStaleCacheOrRethrow(ex: OrdinisUnavailableException): List { + val staleCache = cachedPlatforms + if (staleCache != null) { + log.warn( + "Ordinis classifier is unavailable. Using stale platform cache. platformsUrl={}, cachedCount={}, cacheExpiredAt={}, status={}, errorType={}, errorMessage={}", + classifierProperties.platformsUrl, + staleCache.platforms.size, + staleCache.expiresAt, + ex.statusCode, + rootErrorType(ex), + rootErrorMessage(ex) + ) + return staleCache.platforms + } + + log.error( + "Ordinis classifier is unavailable and no platform cache is available. Platform sync cannot continue. platformsUrl={}, status={}, errorType={}, errorMessage={}", + classifierProperties.platformsUrl, + ex.statusCode, + rootErrorType(ex), + rootErrorMessage(ex), + ex + ) + throw ex } private fun cachePlatforms(platformDtos: List, now: Instant): List { @@ -103,4 +138,10 @@ class PlatformService( ) { fun isActual(now: Instant): Boolean = now.isBefore(expiresAt) } + + private fun rootErrorType(ex: OrdinisUnavailableException): String = + ex.cause?.javaClass?.simpleName ?: ex.javaClass.simpleName + + private fun rootErrorMessage(ex: OrdinisUnavailableException): String? = + ex.cause?.message ?: ex.message } diff --git a/services/pcp-tgu-service/src/main/resources/application.yml b/services/pcp-tgu-service/src/main/resources/application.yml index 63ccfdf..351df1a 100644 --- a/services/pcp-tgu-service/src/main/resources/application.yml +++ b/services/pcp-tgu-service/src/main/resources/application.yml @@ -31,3 +31,9 @@ topics: tgu: test-controller: enabled: false + +management: + endpoints: + web: + exposure: + include: health,info,metrics,prometheus diff --git a/services/pcp-tgu-service/src/test/kotlin/space/nstart/pcp_tgu_service/integration/api/ExternalPointsClientTest.kt b/services/pcp-tgu-service/src/test/kotlin/space/nstart/pcp_tgu_service/integration/api/ExternalPointsClientTest.kt index 9b8b95c..df21568 100644 --- a/services/pcp-tgu-service/src/test/kotlin/space/nstart/pcp_tgu_service/integration/api/ExternalPointsClientTest.kt +++ b/services/pcp-tgu-service/src/test/kotlin/space/nstart/pcp_tgu_service/integration/api/ExternalPointsClientTest.kt @@ -2,14 +2,15 @@ package space.nstart.pcp_tgu_service.integration.api import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.sun.net.httpserver.HttpServer +import io.micrometer.core.instrument.simple.SimpleMeterRegistry import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test import org.mockito.Mockito.mock -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import space.nstart.pcp_tgu_service.config.ClassifierProperties import space.nstart.pcp_tgu_service.config.ExternalApiProperties import space.nstart.pcp_tgu_service.config.PlanningProperties import space.nstart.pcp_tgu_service.domain.PlatformCatalogItem +import space.nstart.pcp_tgu_service.service.OrdinisAvailabilityMetrics import space.nstart.pcp_tgu_service.service.PlatformService import java.net.InetSocketAddress import java.time.Instant @@ -59,7 +60,8 @@ class ExternalPointsClientTest { platformsCacheTtl = java.time.Duration.ofMinutes(15), allowedPlatformStatuses = setOf("OPERATIONAL", "STANDBY") ), - jacksonObjectMapper().findAndRegisterModules() + jacksonObjectMapper().findAndRegisterModules(), + OrdinisAvailabilityMetrics(SimpleMeterRegistry()) ) { override fun loadPlatforms(status: String?): List = error("ExternalPointsClient must use loadAllowedPlatforms") diff --git a/services/pcp-tgu-service/src/test/kotlin/space/nstart/pcp_tgu_service/integration/api/PlatformsClassifierClientTest.kt b/services/pcp-tgu-service/src/test/kotlin/space/nstart/pcp_tgu_service/integration/api/PlatformsClassifierClientTest.kt index f50f791..c385e52 100644 --- a/services/pcp-tgu-service/src/test/kotlin/space/nstart/pcp_tgu_service/integration/api/PlatformsClassifierClientTest.kt +++ b/services/pcp-tgu-service/src/test/kotlin/space/nstart/pcp_tgu_service/integration/api/PlatformsClassifierClientTest.kt @@ -56,7 +56,7 @@ class PlatformsClassifierClientTest { } @Test - fun `fetchPlatforms throws clear error on empty body`() { + fun `fetchPlatforms reports Ordinis unavailability on empty body`() { withServer("") { baseUrl -> val client = PlatformsClassifierClient( ClassifierProperties( @@ -66,20 +66,41 @@ class PlatformsClassifierClientTest { ) ) - val exception = assertThrows(IllegalStateException::class.java) { + val exception = assertThrows(OrdinisUnavailableException::class.java) { client.fetchPlatforms() } - assertEquals("Classifier response body is empty: $baseUrl", exception.message) + assertEquals(baseUrl, exception.classifierUrl) + assertEquals("Classifier response body is empty: $baseUrl", exception.cause?.message) } } - private fun withServer(responseBody: String, block: (String) -> Unit) { + @Test + fun `fetchPlatforms reports Ordinis unavailability on response error`() { + withServer("""{"error":"down"}""", status = 503) { baseUrl -> + val client = PlatformsClassifierClient( + ClassifierProperties( + platformsUrl = baseUrl, + platformsCacheTtl = Duration.ofMinutes(15), + allowedPlatformStatuses = setOf("OPERATIONAL", "STANDBY") + ) + ) + + val exception = assertThrows(OrdinisUnavailableException::class.java) { + client.fetchPlatforms() + } + + assertEquals(baseUrl, exception.classifierUrl) + assertEquals(503, exception.statusCode) + } + } + + private fun withServer(responseBody: String, status: Int = 200, block: (String) -> Unit) { val server = HttpServer.create(InetSocketAddress(0), 0) server.createContext("/") { exchange -> val responseBytes = responseBody.toByteArray() exchange.responseHeaders.add("Content-Type", "application/json") - exchange.sendResponseHeaders(200, responseBytes.size.toLong()) + exchange.sendResponseHeaders(status, responseBytes.size.toLong()) exchange.responseBody.use { body -> body.write(responseBytes) } } server.start() diff --git a/services/pcp-tgu-service/src/test/kotlin/space/nstart/pcp_tgu_service/service/PlatformServiceTest.kt b/services/pcp-tgu-service/src/test/kotlin/space/nstart/pcp_tgu_service/service/PlatformServiceTest.kt index 9c5f16d..f1e7a81 100644 --- a/services/pcp-tgu-service/src/test/kotlin/space/nstart/pcp_tgu_service/service/PlatformServiceTest.kt +++ b/services/pcp-tgu-service/src/test/kotlin/space/nstart/pcp_tgu_service/service/PlatformServiceTest.kt @@ -1,7 +1,9 @@ package space.nstart.pcp_tgu_service.service import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.micrometer.core.instrument.simple.SimpleMeterRegistry import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertThrows import org.junit.jupiter.api.Test import org.mockito.Mockito.mock import org.mockito.Mockito.times @@ -10,6 +12,7 @@ import org.mockito.Mockito.`when` import space.nstart.pcp_tgu_service.config.ClassifierProperties import space.nstart.pcp_tgu_service.dto.NsiPlatformDataDto import space.nstart.pcp_tgu_service.dto.NsiPlatformDto +import space.nstart.pcp_tgu_service.integration.api.OrdinisUnavailableException import space.nstart.pcp_tgu_service.integration.api.PlatformsClassifierClient import java.time.Clock import java.time.Duration @@ -96,6 +99,50 @@ class PlatformServiceTest { verify(client, times(2)).fetchPlatforms() } + @Test + fun `loadPlatforms keeps stale cache when Ordinis is unavailable`() { + val clock = MutableClock(Instant.parse("2026-05-26T12:00:00Z")) + val meterRegistry = SimpleMeterRegistry() + val client = mock(PlatformsClassifierClient::class.java) + `when`(client.fetchPlatforms()) + .thenReturn(listOf(platformDto(id = "cached-id", status = "OPERATIONAL", noradId = 1L))) + .thenThrow(ordinisUnavailable()) + val service = service( + client = client, + clock = clock, + ordinisAvailabilityMetrics = OrdinisAvailabilityMetrics(meterRegistry) + ) + + val firstCall = service.loadPlatforms() + clock.advance(Duration.ofMinutes(15)) + val staleCall = service.loadPlatforms() + + assertEquals(listOf("cached-id"), firstCall.map { it.nsiId }) + assertEquals(listOf("cached-id"), staleCall.map { it.nsiId }) + assertEquals(1.0, meterRegistry.get(OrdinisAvailabilityMetrics.UNAVAILABLE).gauge().value()) + assertEquals(1.0, meterRegistry.get(OrdinisAvailabilityMetrics.UNAVAILABLE_TOTAL).counter().count()) + verify(client, times(2)).fetchPlatforms() + } + + @Test + fun `loadPlatforms rethrows Ordinis outage when cache is empty`() { + val meterRegistry = SimpleMeterRegistry() + val client = mock(PlatformsClassifierClient::class.java) + `when`(client.fetchPlatforms()).thenThrow(ordinisUnavailable()) + val service = service( + client = client, + ordinisAvailabilityMetrics = OrdinisAvailabilityMetrics(meterRegistry) + ) + + assertThrows(OrdinisUnavailableException::class.java) { + service.loadPlatforms() + } + + assertEquals(1.0, meterRegistry.get(OrdinisAvailabilityMetrics.UNAVAILABLE).gauge().value()) + assertEquals(1.0, meterRegistry.get(OrdinisAvailabilityMetrics.UNAVAILABLE_TOTAL).counter().count()) + verify(client, times(1)).fetchPlatforms() + } + @Test fun `loadAllowedPlatforms uses cached platforms and configured filtering`() { val now = Instant.parse("2026-05-26T12:00:00Z") @@ -159,7 +206,8 @@ class PlatformServiceTest { private fun service( client: PlatformsClassifierClient, - clock: Clock = Clock.fixed(Instant.parse("2026-05-26T12:00:00Z"), ZoneOffset.UTC) + clock: Clock = Clock.fixed(Instant.parse("2026-05-26T12:00:00Z"), ZoneOffset.UTC), + ordinisAvailabilityMetrics: OrdinisAvailabilityMetrics = OrdinisAvailabilityMetrics(SimpleMeterRegistry()) ): PlatformService = PlatformService( platformsClassifierClient = client, @@ -169,9 +217,17 @@ class PlatformServiceTest { allowedPlatformStatuses = setOf("OPERATIONAL") ), objectMapper = jacksonObjectMapper().findAndRegisterModules(), + ordinisAvailabilityMetrics = ordinisAvailabilityMetrics, clock = clock ) + private fun ordinisUnavailable(): OrdinisUnavailableException = + OrdinisUnavailableException( + classifierUrl = "http://localhost", + statusCode = null, + cause = RuntimeException("Connection refused") + ) + private fun platformDto( id: String?, status: String,