Handle Ordinis outage in TGU service

Classify Ordinis classifier transport failures and empty responses as unavailable, keep stale platform cache when possible, and expose Prometheus metrics/alerting for the outage signal.
This commit is contained in:
Дмитрий Соловьев
2026-05-30 21:04:57 +03:00
parent 82eddedc92
commit 39f108e496
11 changed files with 235 additions and 16 deletions
@@ -20,6 +20,7 @@ dependencies {
// Spring Boot
implementation("org.springframework.boot:spring-boot-starter-webflux")
implementation("org.springframework.boot:spring-boot-starter-actuator")
runtimeOnly("io.micrometer:micrometer-registry-prometheus")
implementation("org.springdoc:springdoc-openapi-starter-webflux-ui:3.0.0")
implementation("org.springframework.boot:spring-boot-starter-data-jpa")
implementation("org.springframework.boot:spring-boot-starter-flyway")
@@ -5,6 +5,8 @@ import space.nstart.pcp_tgu_service.config.ClassifierProperties
import space.nstart.pcp_tgu_service.dto.NsiPlatformDto
import org.springframework.stereotype.Component
import org.springframework.web.reactive.function.client.WebClient
import org.springframework.web.reactive.function.client.WebClientException
import org.springframework.web.reactive.function.client.WebClientResponseException
/**
* HTTP-клиент для получения списка космических аппаратов из классификатора.
@@ -22,10 +24,33 @@ class PlatformsClassifierClient(
* Загружает список платформ из классификатора НСИ.
*/
fun fetchPlatforms(): List<NsiPlatformDto> =
webClient.get()
.retrieve()
.bodyToMono(Array<NsiPlatformDto>::class.java)
.map { platforms -> platforms.toList() }
.block()
?: throw IllegalStateException("Classifier response body is empty: ${classifierProperties.platformsUrl}")
try {
val platforms = webClient.get()
.retrieve()
.bodyToMono(Array<NsiPlatformDto>::class.java)
.block()
platforms?.toList()
?: throw OrdinisUnavailableException(
classifierUrl = classifierProperties.platformsUrl,
statusCode = null,
cause = IllegalStateException("Classifier response body is empty: ${classifierProperties.platformsUrl}")
)
} catch (ex: WebClientException) {
throw OrdinisUnavailableException(
classifierUrl = classifierProperties.platformsUrl,
statusCode = (ex as? WebClientResponseException)?.statusCode?.value(),
cause = ex
)
}
}
/**
* Семантическая ошибка недоступности Ordinis/НСИ, чтобы service layer мог безопасно
* отличить outage внешней зависимости от ошибок маппинга или доменной логики.
*/
class OrdinisUnavailableException(
val classifierUrl: String,
val statusCode: Int?,
cause: Throwable
) : RuntimeException("Ordinis classifier is unavailable: $classifierUrl", cause)
@@ -0,0 +1,40 @@
package space.nstart.pcp_tgu_service.service
import io.micrometer.core.instrument.Counter
import io.micrometer.core.instrument.Gauge
import io.micrometer.core.instrument.MeterRegistry
import org.springframework.stereotype.Component
import java.util.concurrent.atomic.AtomicInteger
/**
* Метрики доступности Ordinis/НСИ для alerting по последней попытке синхронизации.
*/
@Component
class OrdinisAvailabilityMetrics(
meterRegistry: MeterRegistry
) {
private val unavailableGaugeValue = AtomicInteger(0)
private val unavailableCounter = Counter.builder(UNAVAILABLE_TOTAL)
.description("Total Ordinis classifier fetch failures detected by pcp-tgu-service")
.register(meterRegistry)
init {
Gauge.builder(UNAVAILABLE, unavailableGaugeValue) { value -> value.get().toDouble() }
.description("1 when the last Ordinis classifier fetch failed, 0 after a successful fetch")
.register(meterRegistry)
}
fun markAvailable() {
unavailableGaugeValue.set(0)
}
fun markUnavailable() {
unavailableGaugeValue.set(1)
unavailableCounter.increment()
}
companion object {
const val UNAVAILABLE = "pcp_tgu_ordinis_unavailable"
const val UNAVAILABLE_TOTAL = "pcp_tgu_ordinis_unavailable_total"
}
}
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper
import space.nstart.pcp_tgu_service.config.ClassifierProperties
import space.nstart.pcp_tgu_service.domain.PlatformCatalogItem
import space.nstart.pcp_tgu_service.dto.NsiPlatformDto
import space.nstart.pcp_tgu_service.integration.api.OrdinisUnavailableException
import space.nstart.pcp_tgu_service.integration.api.PlatformsClassifierClient
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Service
@@ -16,6 +17,7 @@ class PlatformService(
private val platformsClassifierClient: PlatformsClassifierClient,
private val classifierProperties: ClassifierProperties,
private val objectMapper: ObjectMapper,
private val ordinisAvailabilityMetrics: OrdinisAvailabilityMetrics,
private val clock: Clock = Clock.systemUTC()
) {
private val log = LoggerFactory.getLogger(javaClass)
@@ -63,7 +65,40 @@ class PlatformService(
}
private fun fetchAndCachePlatforms(now: Instant): List<PlatformCatalogItem> {
return cachePlatforms(platformsClassifierClient.fetchPlatforms(), now)
return try {
val platforms = cachePlatforms(platformsClassifierClient.fetchPlatforms(), now)
ordinisAvailabilityMetrics.markAvailable()
platforms
} catch (ex: OrdinisUnavailableException) {
ordinisAvailabilityMetrics.markUnavailable()
useStaleCacheOrRethrow(ex)
}
}
private fun useStaleCacheOrRethrow(ex: OrdinisUnavailableException): List<PlatformCatalogItem> {
val staleCache = cachedPlatforms
if (staleCache != null) {
log.warn(
"Ordinis classifier is unavailable. Using stale platform cache. platformsUrl={}, cachedCount={}, cacheExpiredAt={}, status={}, errorType={}, errorMessage={}",
classifierProperties.platformsUrl,
staleCache.platforms.size,
staleCache.expiresAt,
ex.statusCode,
rootErrorType(ex),
rootErrorMessage(ex)
)
return staleCache.platforms
}
log.error(
"Ordinis classifier is unavailable and no platform cache is available. Platform sync cannot continue. platformsUrl={}, status={}, errorType={}, errorMessage={}",
classifierProperties.platformsUrl,
ex.statusCode,
rootErrorType(ex),
rootErrorMessage(ex),
ex
)
throw ex
}
private fun cachePlatforms(platformDtos: List<NsiPlatformDto>, now: Instant): List<PlatformCatalogItem> {
@@ -103,4 +138,10 @@ class PlatformService(
) {
fun isActual(now: Instant): Boolean = now.isBefore(expiresAt)
}
private fun rootErrorType(ex: OrdinisUnavailableException): String =
ex.cause?.javaClass?.simpleName ?: ex.javaClass.simpleName
private fun rootErrorMessage(ex: OrdinisUnavailableException): String? =
ex.cause?.message ?: ex.message
}
@@ -31,3 +31,9 @@ topics:
tgu:
test-controller:
enabled: false
management:
endpoints:
web:
exposure:
include: health,info,metrics,prometheus
@@ -2,14 +2,15 @@ package space.nstart.pcp_tgu_service.integration.api
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.sun.net.httpserver.HttpServer
import io.micrometer.core.instrument.simple.SimpleMeterRegistry
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
import org.mockito.Mockito.mock
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import space.nstart.pcp_tgu_service.config.ClassifierProperties
import space.nstart.pcp_tgu_service.config.ExternalApiProperties
import space.nstart.pcp_tgu_service.config.PlanningProperties
import space.nstart.pcp_tgu_service.domain.PlatformCatalogItem
import space.nstart.pcp_tgu_service.service.OrdinisAvailabilityMetrics
import space.nstart.pcp_tgu_service.service.PlatformService
import java.net.InetSocketAddress
import java.time.Instant
@@ -59,7 +60,8 @@ class ExternalPointsClientTest {
platformsCacheTtl = java.time.Duration.ofMinutes(15),
allowedPlatformStatuses = setOf("OPERATIONAL", "STANDBY")
),
jacksonObjectMapper().findAndRegisterModules()
jacksonObjectMapper().findAndRegisterModules(),
OrdinisAvailabilityMetrics(SimpleMeterRegistry())
) {
override fun loadPlatforms(status: String?): List<PlatformCatalogItem> =
error("ExternalPointsClient must use loadAllowedPlatforms")
@@ -56,7 +56,7 @@ class PlatformsClassifierClientTest {
}
@Test
fun `fetchPlatforms throws clear error on empty body`() {
fun `fetchPlatforms reports Ordinis unavailability on empty body`() {
withServer("") { baseUrl ->
val client = PlatformsClassifierClient(
ClassifierProperties(
@@ -66,20 +66,41 @@ class PlatformsClassifierClientTest {
)
)
val exception = assertThrows(IllegalStateException::class.java) {
val exception = assertThrows(OrdinisUnavailableException::class.java) {
client.fetchPlatforms()
}
assertEquals("Classifier response body is empty: $baseUrl", exception.message)
assertEquals(baseUrl, exception.classifierUrl)
assertEquals("Classifier response body is empty: $baseUrl", exception.cause?.message)
}
}
private fun withServer(responseBody: String, block: (String) -> Unit) {
@Test
fun `fetchPlatforms reports Ordinis unavailability on response error`() {
withServer("""{"error":"down"}""", status = 503) { baseUrl ->
val client = PlatformsClassifierClient(
ClassifierProperties(
platformsUrl = baseUrl,
platformsCacheTtl = Duration.ofMinutes(15),
allowedPlatformStatuses = setOf("OPERATIONAL", "STANDBY")
)
)
val exception = assertThrows(OrdinisUnavailableException::class.java) {
client.fetchPlatforms()
}
assertEquals(baseUrl, exception.classifierUrl)
assertEquals(503, exception.statusCode)
}
}
private fun withServer(responseBody: String, status: Int = 200, block: (String) -> Unit) {
val server = HttpServer.create(InetSocketAddress(0), 0)
server.createContext("/") { exchange ->
val responseBytes = responseBody.toByteArray()
exchange.responseHeaders.add("Content-Type", "application/json")
exchange.sendResponseHeaders(200, responseBytes.size.toLong())
exchange.sendResponseHeaders(status, responseBytes.size.toLong())
exchange.responseBody.use { body -> body.write(responseBytes) }
}
server.start()
@@ -1,7 +1,9 @@
package space.nstart.pcp_tgu_service.service
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import io.micrometer.core.instrument.simple.SimpleMeterRegistry
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertThrows
import org.junit.jupiter.api.Test
import org.mockito.Mockito.mock
import org.mockito.Mockito.times
@@ -10,6 +12,7 @@ import org.mockito.Mockito.`when`
import space.nstart.pcp_tgu_service.config.ClassifierProperties
import space.nstart.pcp_tgu_service.dto.NsiPlatformDataDto
import space.nstart.pcp_tgu_service.dto.NsiPlatformDto
import space.nstart.pcp_tgu_service.integration.api.OrdinisUnavailableException
import space.nstart.pcp_tgu_service.integration.api.PlatformsClassifierClient
import java.time.Clock
import java.time.Duration
@@ -96,6 +99,50 @@ class PlatformServiceTest {
verify(client, times(2)).fetchPlatforms()
}
@Test
fun `loadPlatforms keeps stale cache when Ordinis is unavailable`() {
val clock = MutableClock(Instant.parse("2026-05-26T12:00:00Z"))
val meterRegistry = SimpleMeterRegistry()
val client = mock(PlatformsClassifierClient::class.java)
`when`(client.fetchPlatforms())
.thenReturn(listOf(platformDto(id = "cached-id", status = "OPERATIONAL", noradId = 1L)))
.thenThrow(ordinisUnavailable())
val service = service(
client = client,
clock = clock,
ordinisAvailabilityMetrics = OrdinisAvailabilityMetrics(meterRegistry)
)
val firstCall = service.loadPlatforms()
clock.advance(Duration.ofMinutes(15))
val staleCall = service.loadPlatforms()
assertEquals(listOf("cached-id"), firstCall.map { it.nsiId })
assertEquals(listOf("cached-id"), staleCall.map { it.nsiId })
assertEquals(1.0, meterRegistry.get(OrdinisAvailabilityMetrics.UNAVAILABLE).gauge().value())
assertEquals(1.0, meterRegistry.get(OrdinisAvailabilityMetrics.UNAVAILABLE_TOTAL).counter().count())
verify(client, times(2)).fetchPlatforms()
}
@Test
fun `loadPlatforms rethrows Ordinis outage when cache is empty`() {
val meterRegistry = SimpleMeterRegistry()
val client = mock(PlatformsClassifierClient::class.java)
`when`(client.fetchPlatforms()).thenThrow(ordinisUnavailable())
val service = service(
client = client,
ordinisAvailabilityMetrics = OrdinisAvailabilityMetrics(meterRegistry)
)
assertThrows(OrdinisUnavailableException::class.java) {
service.loadPlatforms()
}
assertEquals(1.0, meterRegistry.get(OrdinisAvailabilityMetrics.UNAVAILABLE).gauge().value())
assertEquals(1.0, meterRegistry.get(OrdinisAvailabilityMetrics.UNAVAILABLE_TOTAL).counter().count())
verify(client, times(1)).fetchPlatforms()
}
@Test
fun `loadAllowedPlatforms uses cached platforms and configured filtering`() {
val now = Instant.parse("2026-05-26T12:00:00Z")
@@ -159,7 +206,8 @@ class PlatformServiceTest {
private fun service(
client: PlatformsClassifierClient,
clock: Clock = Clock.fixed(Instant.parse("2026-05-26T12:00:00Z"), ZoneOffset.UTC)
clock: Clock = Clock.fixed(Instant.parse("2026-05-26T12:00:00Z"), ZoneOffset.UTC),
ordinisAvailabilityMetrics: OrdinisAvailabilityMetrics = OrdinisAvailabilityMetrics(SimpleMeterRegistry())
): PlatformService =
PlatformService(
platformsClassifierClient = client,
@@ -169,9 +217,17 @@ class PlatformServiceTest {
allowedPlatformStatuses = setOf("OPERATIONAL")
),
objectMapper = jacksonObjectMapper().findAndRegisterModules(),
ordinisAvailabilityMetrics = ordinisAvailabilityMetrics,
clock = clock
)
private fun ordinisUnavailable(): OrdinisUnavailableException =
OrdinisUnavailableException(
classifierUrl = "http://localhost",
statusCode = null,
cause = RuntimeException("Connection refused")
)
private fun platformDto(
id: String?,
status: String,