Handle Ordinis outage in TGU service
Classify Ordinis classifier transport failures and empty responses as unavailable, keep stale platform cache when possible, and expose Prometheus metrics/alerting for the outage signal.
This commit is contained in:
@@ -20,6 +20,7 @@ dependencies {
|
||||
// Spring Boot
|
||||
implementation("org.springframework.boot:spring-boot-starter-webflux")
|
||||
implementation("org.springframework.boot:spring-boot-starter-actuator")
|
||||
runtimeOnly("io.micrometer:micrometer-registry-prometheus")
|
||||
implementation("org.springdoc:springdoc-openapi-starter-webflux-ui:3.0.0")
|
||||
implementation("org.springframework.boot:spring-boot-starter-data-jpa")
|
||||
implementation("org.springframework.boot:spring-boot-starter-flyway")
|
||||
|
||||
+31
-6
@@ -5,6 +5,8 @@ import space.nstart.pcp_tgu_service.config.ClassifierProperties
|
||||
import space.nstart.pcp_tgu_service.dto.NsiPlatformDto
|
||||
import org.springframework.stereotype.Component
|
||||
import org.springframework.web.reactive.function.client.WebClient
|
||||
import org.springframework.web.reactive.function.client.WebClientException
|
||||
import org.springframework.web.reactive.function.client.WebClientResponseException
|
||||
|
||||
/**
|
||||
* HTTP-клиент для получения списка космических аппаратов из классификатора.
|
||||
@@ -22,10 +24,33 @@ class PlatformsClassifierClient(
|
||||
* Загружает список платформ из классификатора НСИ.
|
||||
*/
|
||||
fun fetchPlatforms(): List<NsiPlatformDto> =
|
||||
webClient.get()
|
||||
.retrieve()
|
||||
.bodyToMono(Array<NsiPlatformDto>::class.java)
|
||||
.map { platforms -> platforms.toList() }
|
||||
.block()
|
||||
?: throw IllegalStateException("Classifier response body is empty: ${classifierProperties.platformsUrl}")
|
||||
try {
|
||||
val platforms = webClient.get()
|
||||
.retrieve()
|
||||
.bodyToMono(Array<NsiPlatformDto>::class.java)
|
||||
.block()
|
||||
|
||||
platforms?.toList()
|
||||
?: throw OrdinisUnavailableException(
|
||||
classifierUrl = classifierProperties.platformsUrl,
|
||||
statusCode = null,
|
||||
cause = IllegalStateException("Classifier response body is empty: ${classifierProperties.platformsUrl}")
|
||||
)
|
||||
} catch (ex: WebClientException) {
|
||||
throw OrdinisUnavailableException(
|
||||
classifierUrl = classifierProperties.platformsUrl,
|
||||
statusCode = (ex as? WebClientResponseException)?.statusCode?.value(),
|
||||
cause = ex
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Семантическая ошибка недоступности Ordinis/НСИ, чтобы service layer мог безопасно
|
||||
* отличить outage внешней зависимости от ошибок маппинга или доменной логики.
|
||||
*/
|
||||
class OrdinisUnavailableException(
|
||||
val classifierUrl: String,
|
||||
val statusCode: Int?,
|
||||
cause: Throwable
|
||||
) : RuntimeException("Ordinis classifier is unavailable: $classifierUrl", cause)
|
||||
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
package space.nstart.pcp_tgu_service.service
|
||||
|
||||
import io.micrometer.core.instrument.Counter
|
||||
import io.micrometer.core.instrument.Gauge
|
||||
import io.micrometer.core.instrument.MeterRegistry
|
||||
import org.springframework.stereotype.Component
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
/**
|
||||
* Метрики доступности Ordinis/НСИ для alerting по последней попытке синхронизации.
|
||||
*/
|
||||
@Component
|
||||
class OrdinisAvailabilityMetrics(
|
||||
meterRegistry: MeterRegistry
|
||||
) {
|
||||
private val unavailableGaugeValue = AtomicInteger(0)
|
||||
private val unavailableCounter = Counter.builder(UNAVAILABLE_TOTAL)
|
||||
.description("Total Ordinis classifier fetch failures detected by pcp-tgu-service")
|
||||
.register(meterRegistry)
|
||||
|
||||
init {
|
||||
Gauge.builder(UNAVAILABLE, unavailableGaugeValue) { value -> value.get().toDouble() }
|
||||
.description("1 when the last Ordinis classifier fetch failed, 0 after a successful fetch")
|
||||
.register(meterRegistry)
|
||||
}
|
||||
|
||||
fun markAvailable() {
|
||||
unavailableGaugeValue.set(0)
|
||||
}
|
||||
|
||||
fun markUnavailable() {
|
||||
unavailableGaugeValue.set(1)
|
||||
unavailableCounter.increment()
|
||||
}
|
||||
|
||||
companion object {
|
||||
const val UNAVAILABLE = "pcp_tgu_ordinis_unavailable"
|
||||
const val UNAVAILABLE_TOTAL = "pcp_tgu_ordinis_unavailable_total"
|
||||
}
|
||||
}
|
||||
+42
-1
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper
|
||||
import space.nstart.pcp_tgu_service.config.ClassifierProperties
|
||||
import space.nstart.pcp_tgu_service.domain.PlatformCatalogItem
|
||||
import space.nstart.pcp_tgu_service.dto.NsiPlatformDto
|
||||
import space.nstart.pcp_tgu_service.integration.api.OrdinisUnavailableException
|
||||
import space.nstart.pcp_tgu_service.integration.api.PlatformsClassifierClient
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.stereotype.Service
|
||||
@@ -16,6 +17,7 @@ class PlatformService(
|
||||
private val platformsClassifierClient: PlatformsClassifierClient,
|
||||
private val classifierProperties: ClassifierProperties,
|
||||
private val objectMapper: ObjectMapper,
|
||||
private val ordinisAvailabilityMetrics: OrdinisAvailabilityMetrics,
|
||||
private val clock: Clock = Clock.systemUTC()
|
||||
) {
|
||||
private val log = LoggerFactory.getLogger(javaClass)
|
||||
@@ -63,7 +65,40 @@ class PlatformService(
|
||||
}
|
||||
|
||||
private fun fetchAndCachePlatforms(now: Instant): List<PlatformCatalogItem> {
|
||||
return cachePlatforms(platformsClassifierClient.fetchPlatforms(), now)
|
||||
return try {
|
||||
val platforms = cachePlatforms(platformsClassifierClient.fetchPlatforms(), now)
|
||||
ordinisAvailabilityMetrics.markAvailable()
|
||||
platforms
|
||||
} catch (ex: OrdinisUnavailableException) {
|
||||
ordinisAvailabilityMetrics.markUnavailable()
|
||||
useStaleCacheOrRethrow(ex)
|
||||
}
|
||||
}
|
||||
|
||||
private fun useStaleCacheOrRethrow(ex: OrdinisUnavailableException): List<PlatformCatalogItem> {
|
||||
val staleCache = cachedPlatforms
|
||||
if (staleCache != null) {
|
||||
log.warn(
|
||||
"Ordinis classifier is unavailable. Using stale platform cache. platformsUrl={}, cachedCount={}, cacheExpiredAt={}, status={}, errorType={}, errorMessage={}",
|
||||
classifierProperties.platformsUrl,
|
||||
staleCache.platforms.size,
|
||||
staleCache.expiresAt,
|
||||
ex.statusCode,
|
||||
rootErrorType(ex),
|
||||
rootErrorMessage(ex)
|
||||
)
|
||||
return staleCache.platforms
|
||||
}
|
||||
|
||||
log.error(
|
||||
"Ordinis classifier is unavailable and no platform cache is available. Platform sync cannot continue. platformsUrl={}, status={}, errorType={}, errorMessage={}",
|
||||
classifierProperties.platformsUrl,
|
||||
ex.statusCode,
|
||||
rootErrorType(ex),
|
||||
rootErrorMessage(ex),
|
||||
ex
|
||||
)
|
||||
throw ex
|
||||
}
|
||||
|
||||
private fun cachePlatforms(platformDtos: List<NsiPlatformDto>, now: Instant): List<PlatformCatalogItem> {
|
||||
@@ -103,4 +138,10 @@ class PlatformService(
|
||||
) {
|
||||
fun isActual(now: Instant): Boolean = now.isBefore(expiresAt)
|
||||
}
|
||||
|
||||
private fun rootErrorType(ex: OrdinisUnavailableException): String =
|
||||
ex.cause?.javaClass?.simpleName ?: ex.javaClass.simpleName
|
||||
|
||||
private fun rootErrorMessage(ex: OrdinisUnavailableException): String? =
|
||||
ex.cause?.message ?: ex.message
|
||||
}
|
||||
|
||||
@@ -31,3 +31,9 @@ topics:
|
||||
tgu:
|
||||
test-controller:
|
||||
enabled: false
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: health,info,metrics,prometheus
|
||||
|
||||
+4
-2
@@ -2,14 +2,15 @@ package space.nstart.pcp_tgu_service.integration.api
|
||||
|
||||
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
|
||||
import com.sun.net.httpserver.HttpServer
|
||||
import io.micrometer.core.instrument.simple.SimpleMeterRegistry
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Test
|
||||
import org.mockito.Mockito.mock
|
||||
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
|
||||
import space.nstart.pcp_tgu_service.config.ClassifierProperties
|
||||
import space.nstart.pcp_tgu_service.config.ExternalApiProperties
|
||||
import space.nstart.pcp_tgu_service.config.PlanningProperties
|
||||
import space.nstart.pcp_tgu_service.domain.PlatformCatalogItem
|
||||
import space.nstart.pcp_tgu_service.service.OrdinisAvailabilityMetrics
|
||||
import space.nstart.pcp_tgu_service.service.PlatformService
|
||||
import java.net.InetSocketAddress
|
||||
import java.time.Instant
|
||||
@@ -59,7 +60,8 @@ class ExternalPointsClientTest {
|
||||
platformsCacheTtl = java.time.Duration.ofMinutes(15),
|
||||
allowedPlatformStatuses = setOf("OPERATIONAL", "STANDBY")
|
||||
),
|
||||
jacksonObjectMapper().findAndRegisterModules()
|
||||
jacksonObjectMapper().findAndRegisterModules(),
|
||||
OrdinisAvailabilityMetrics(SimpleMeterRegistry())
|
||||
) {
|
||||
override fun loadPlatforms(status: String?): List<PlatformCatalogItem> =
|
||||
error("ExternalPointsClient must use loadAllowedPlatforms")
|
||||
|
||||
+26
-5
@@ -56,7 +56,7 @@ class PlatformsClassifierClientTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `fetchPlatforms throws clear error on empty body`() {
|
||||
fun `fetchPlatforms reports Ordinis unavailability on empty body`() {
|
||||
withServer("") { baseUrl ->
|
||||
val client = PlatformsClassifierClient(
|
||||
ClassifierProperties(
|
||||
@@ -66,20 +66,41 @@ class PlatformsClassifierClientTest {
|
||||
)
|
||||
)
|
||||
|
||||
val exception = assertThrows(IllegalStateException::class.java) {
|
||||
val exception = assertThrows(OrdinisUnavailableException::class.java) {
|
||||
client.fetchPlatforms()
|
||||
}
|
||||
|
||||
assertEquals("Classifier response body is empty: $baseUrl", exception.message)
|
||||
assertEquals(baseUrl, exception.classifierUrl)
|
||||
assertEquals("Classifier response body is empty: $baseUrl", exception.cause?.message)
|
||||
}
|
||||
}
|
||||
|
||||
private fun withServer(responseBody: String, block: (String) -> Unit) {
|
||||
@Test
|
||||
fun `fetchPlatforms reports Ordinis unavailability on response error`() {
|
||||
withServer("""{"error":"down"}""", status = 503) { baseUrl ->
|
||||
val client = PlatformsClassifierClient(
|
||||
ClassifierProperties(
|
||||
platformsUrl = baseUrl,
|
||||
platformsCacheTtl = Duration.ofMinutes(15),
|
||||
allowedPlatformStatuses = setOf("OPERATIONAL", "STANDBY")
|
||||
)
|
||||
)
|
||||
|
||||
val exception = assertThrows(OrdinisUnavailableException::class.java) {
|
||||
client.fetchPlatforms()
|
||||
}
|
||||
|
||||
assertEquals(baseUrl, exception.classifierUrl)
|
||||
assertEquals(503, exception.statusCode)
|
||||
}
|
||||
}
|
||||
|
||||
private fun withServer(responseBody: String, status: Int = 200, block: (String) -> Unit) {
|
||||
val server = HttpServer.create(InetSocketAddress(0), 0)
|
||||
server.createContext("/") { exchange ->
|
||||
val responseBytes = responseBody.toByteArray()
|
||||
exchange.responseHeaders.add("Content-Type", "application/json")
|
||||
exchange.sendResponseHeaders(200, responseBytes.size.toLong())
|
||||
exchange.sendResponseHeaders(status, responseBytes.size.toLong())
|
||||
exchange.responseBody.use { body -> body.write(responseBytes) }
|
||||
}
|
||||
server.start()
|
||||
|
||||
+57
-1
@@ -1,7 +1,9 @@
|
||||
package space.nstart.pcp_tgu_service.service
|
||||
|
||||
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
|
||||
import io.micrometer.core.instrument.simple.SimpleMeterRegistry
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertThrows
|
||||
import org.junit.jupiter.api.Test
|
||||
import org.mockito.Mockito.mock
|
||||
import org.mockito.Mockito.times
|
||||
@@ -10,6 +12,7 @@ import org.mockito.Mockito.`when`
|
||||
import space.nstart.pcp_tgu_service.config.ClassifierProperties
|
||||
import space.nstart.pcp_tgu_service.dto.NsiPlatformDataDto
|
||||
import space.nstart.pcp_tgu_service.dto.NsiPlatformDto
|
||||
import space.nstart.pcp_tgu_service.integration.api.OrdinisUnavailableException
|
||||
import space.nstart.pcp_tgu_service.integration.api.PlatformsClassifierClient
|
||||
import java.time.Clock
|
||||
import java.time.Duration
|
||||
@@ -96,6 +99,50 @@ class PlatformServiceTest {
|
||||
verify(client, times(2)).fetchPlatforms()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `loadPlatforms keeps stale cache when Ordinis is unavailable`() {
|
||||
val clock = MutableClock(Instant.parse("2026-05-26T12:00:00Z"))
|
||||
val meterRegistry = SimpleMeterRegistry()
|
||||
val client = mock(PlatformsClassifierClient::class.java)
|
||||
`when`(client.fetchPlatforms())
|
||||
.thenReturn(listOf(platformDto(id = "cached-id", status = "OPERATIONAL", noradId = 1L)))
|
||||
.thenThrow(ordinisUnavailable())
|
||||
val service = service(
|
||||
client = client,
|
||||
clock = clock,
|
||||
ordinisAvailabilityMetrics = OrdinisAvailabilityMetrics(meterRegistry)
|
||||
)
|
||||
|
||||
val firstCall = service.loadPlatforms()
|
||||
clock.advance(Duration.ofMinutes(15))
|
||||
val staleCall = service.loadPlatforms()
|
||||
|
||||
assertEquals(listOf("cached-id"), firstCall.map { it.nsiId })
|
||||
assertEquals(listOf("cached-id"), staleCall.map { it.nsiId })
|
||||
assertEquals(1.0, meterRegistry.get(OrdinisAvailabilityMetrics.UNAVAILABLE).gauge().value())
|
||||
assertEquals(1.0, meterRegistry.get(OrdinisAvailabilityMetrics.UNAVAILABLE_TOTAL).counter().count())
|
||||
verify(client, times(2)).fetchPlatforms()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `loadPlatforms rethrows Ordinis outage when cache is empty`() {
|
||||
val meterRegistry = SimpleMeterRegistry()
|
||||
val client = mock(PlatformsClassifierClient::class.java)
|
||||
`when`(client.fetchPlatforms()).thenThrow(ordinisUnavailable())
|
||||
val service = service(
|
||||
client = client,
|
||||
ordinisAvailabilityMetrics = OrdinisAvailabilityMetrics(meterRegistry)
|
||||
)
|
||||
|
||||
assertThrows(OrdinisUnavailableException::class.java) {
|
||||
service.loadPlatforms()
|
||||
}
|
||||
|
||||
assertEquals(1.0, meterRegistry.get(OrdinisAvailabilityMetrics.UNAVAILABLE).gauge().value())
|
||||
assertEquals(1.0, meterRegistry.get(OrdinisAvailabilityMetrics.UNAVAILABLE_TOTAL).counter().count())
|
||||
verify(client, times(1)).fetchPlatforms()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `loadAllowedPlatforms uses cached platforms and configured filtering`() {
|
||||
val now = Instant.parse("2026-05-26T12:00:00Z")
|
||||
@@ -159,7 +206,8 @@ class PlatformServiceTest {
|
||||
|
||||
private fun service(
|
||||
client: PlatformsClassifierClient,
|
||||
clock: Clock = Clock.fixed(Instant.parse("2026-05-26T12:00:00Z"), ZoneOffset.UTC)
|
||||
clock: Clock = Clock.fixed(Instant.parse("2026-05-26T12:00:00Z"), ZoneOffset.UTC),
|
||||
ordinisAvailabilityMetrics: OrdinisAvailabilityMetrics = OrdinisAvailabilityMetrics(SimpleMeterRegistry())
|
||||
): PlatformService =
|
||||
PlatformService(
|
||||
platformsClassifierClient = client,
|
||||
@@ -169,9 +217,17 @@ class PlatformServiceTest {
|
||||
allowedPlatformStatuses = setOf("OPERATIONAL")
|
||||
),
|
||||
objectMapper = jacksonObjectMapper().findAndRegisterModules(),
|
||||
ordinisAvailabilityMetrics = ordinisAvailabilityMetrics,
|
||||
clock = clock
|
||||
)
|
||||
|
||||
private fun ordinisUnavailable(): OrdinisUnavailableException =
|
||||
OrdinisUnavailableException(
|
||||
classifierUrl = "http://localhost",
|
||||
statusCode = null,
|
||||
cause = RuntimeException("Connection refused")
|
||||
)
|
||||
|
||||
private fun platformDto(
|
||||
id: String?,
|
||||
status: String,
|
||||
|
||||
Reference in New Issue
Block a user