добавить масштабируемый outbox worker для паспортов маршрутов
- добавить properties и config values для route passport outbox worker - включить Spring scheduling в route-processing service - добавить транзакционный сервис claim и status transitions для outbox - публиковать claimed outbox rows через существующий Kafka publisher - обрабатывать publish failures по каждой row с bounded exponential backoff - обеспечить безопасность при нескольких pod через FOR UPDATE SKIP LOCKED и lockedUntil - добавить unit tests для claim, publish transitions, retry/fail behavior и batch-обработки worker
This commit is contained in:
Дмитрий Соловьев
+44
@@ -0,0 +1,44 @@
|
||||
package space.nstart.pcp.routepassportconsumer.config
|
||||
|
||||
import jakarta.validation.constraints.AssertTrue
|
||||
import jakarta.validation.constraints.Min
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties
|
||||
import org.springframework.validation.annotation.Validated
|
||||
|
||||
/**
|
||||
* Operational settings for the route passport outbox worker.
|
||||
*
|
||||
* Values control scheduling cadence, DB claim size, lock lease time and bounded retry backoff.
|
||||
*/
|
||||
@Validated
|
||||
@ConfigurationProperties(prefix = "app.route-passport.outbox")
|
||||
data class RoutePassportOutboxProperties(
|
||||
|
||||
/** Enables scheduled publication of pending route passport outbox rows. */
|
||||
val enabled: Boolean = true,
|
||||
|
||||
/** Maximum number of due rows claimed by one worker tick. */
|
||||
@field:Min(1)
|
||||
val batchSize: Int = 50,
|
||||
|
||||
/** Delay between scheduled worker runs. */
|
||||
@field:Min(100)
|
||||
val fixedDelayMs: Long = 1_000,
|
||||
|
||||
/** Lease duration for a claimed row; expired locks can be claimed by another instance. */
|
||||
@field:Min(1_000)
|
||||
val lockTimeoutMs: Long = 60_000,
|
||||
|
||||
/** Backoff after the first failed publication attempt. */
|
||||
@field:Min(100)
|
||||
val initialBackoffMs: Long = 1_000,
|
||||
|
||||
/** Maximum backoff between publication attempts. */
|
||||
@field:Min(100)
|
||||
val maxBackoffMs: Long = 60_000,
|
||||
) {
|
||||
|
||||
@get:AssertTrue(message = "maxBackoffMs must be greater than or equal to initialBackoffMs")
|
||||
val isMaxBackoffGreaterThanOrEqualToInitialBackoff: Boolean
|
||||
get() = maxBackoffMs >= initialBackoffMs
|
||||
}
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
package space.nstart.pcp.routepassportconsumer.service
|
||||
|
||||
import org.springframework.stereotype.Service
|
||||
import org.springframework.transaction.annotation.Transactional
|
||||
import space.nstart.pcp.routepassportconsumer.config.RoutePassportOutboxProperties
|
||||
import space.nstart.pcp.routepassportconsumer.entity.RoutePassportOutboxEntity
|
||||
import space.nstart.pcp.routepassportconsumer.entity.RoutePassportOutboxStatus
|
||||
import space.nstart.pcp.routepassportconsumer.repository.RoutePassportOutboxRepository
|
||||
import java.time.Duration
|
||||
import java.time.LocalDateTime
|
||||
import java.util.UUID
|
||||
|
||||
/**
|
||||
* Owns transactional claim and status transitions for route passport outbox rows.
|
||||
*/
|
||||
@Service
|
||||
class RoutePassportOutboxService(
|
||||
private val routePassportOutboxRepository: RoutePassportOutboxRepository,
|
||||
private val outboxProperties: RoutePassportOutboxProperties,
|
||||
) {
|
||||
|
||||
/**
|
||||
* Claims due pending rows in one DB transaction using SELECT FOR UPDATE SKIP LOCKED.
|
||||
*/
|
||||
@Transactional
|
||||
fun claimDueBatch(): List<RoutePassportOutboxEntity> {
|
||||
if (!outboxProperties.enabled) {
|
||||
return emptyList()
|
||||
}
|
||||
|
||||
val now = LocalDateTime.now()
|
||||
val lockedUntil = now.plus(Duration.ofMillis(outboxProperties.lockTimeoutMs))
|
||||
val claimedRows = routePassportOutboxRepository.findDueForUpdateSkipLocked(outboxProperties.batchSize)
|
||||
|
||||
claimedRows.forEach { outbox ->
|
||||
outbox.lockedUntil = lockedUntil
|
||||
outbox.updatedAt = now
|
||||
}
|
||||
|
||||
return routePassportOutboxRepository.saveAll(claimedRows).toList()
|
||||
}
|
||||
|
||||
/**
|
||||
* Marks a successfully published row as terminally published.
|
||||
*/
|
||||
@Transactional
|
||||
fun markPublished(outboxId: UUID) {
|
||||
val now = LocalDateTime.now()
|
||||
val outbox = findOutbox(outboxId)
|
||||
|
||||
outbox.status = RoutePassportOutboxStatus.PUBLISHED
|
||||
outbox.publishedAt = now
|
||||
outbox.lockedUntil = null
|
||||
outbox.lastError = null
|
||||
outbox.updatedAt = now
|
||||
|
||||
routePassportOutboxRepository.save(outbox)
|
||||
}
|
||||
|
||||
/**
|
||||
* Records a publish failure and either schedules bounded retry or marks the row as failed.
|
||||
*/
|
||||
@Transactional
|
||||
fun markRetryOrFailed(outboxId: UUID, exception: Exception) {
|
||||
val now = LocalDateTime.now()
|
||||
val outbox = findOutbox(outboxId)
|
||||
|
||||
outbox.attempts += 1
|
||||
outbox.lastError = compactError(exception)
|
||||
outbox.lockedUntil = null
|
||||
outbox.updatedAt = now
|
||||
|
||||
if (outbox.attempts >= outbox.maxAttempts) {
|
||||
outbox.status = RoutePassportOutboxStatus.FAILED
|
||||
} else {
|
||||
outbox.status = RoutePassportOutboxStatus.PENDING
|
||||
outbox.nextAttemptAt = now.plus(Duration.ofMillis(calculateBackoffMs(outbox.attempts)))
|
||||
}
|
||||
|
||||
routePassportOutboxRepository.save(outbox)
|
||||
}
|
||||
|
||||
private fun findOutbox(outboxId: UUID): RoutePassportOutboxEntity =
|
||||
routePassportOutboxRepository.findById(outboxId)
|
||||
.orElseThrow { IllegalStateException("Route passport outbox row not found: $outboxId") }
|
||||
|
||||
private fun calculateBackoffMs(attempts: Int): Long {
|
||||
var backoffMs = outboxProperties.initialBackoffMs.coerceAtMost(outboxProperties.maxBackoffMs)
|
||||
|
||||
repeat((attempts - 1).coerceAtLeast(0)) {
|
||||
backoffMs = if (backoffMs >= outboxProperties.maxBackoffMs / 2) {
|
||||
outboxProperties.maxBackoffMs
|
||||
} else {
|
||||
(backoffMs * 2).coerceAtMost(outboxProperties.maxBackoffMs)
|
||||
}
|
||||
}
|
||||
|
||||
return backoffMs
|
||||
}
|
||||
|
||||
private fun compactError(exception: Exception): String =
|
||||
"${exception::class.java.name}: ${exception.message ?: ""}".take(MAX_ERROR_LENGTH)
|
||||
|
||||
private companion object {
|
||||
const val MAX_ERROR_LENGTH = 2_000
|
||||
}
|
||||
}
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
package space.nstart.pcp.routepassportconsumer.service
|
||||
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.scheduling.annotation.Scheduled
|
||||
import org.springframework.stereotype.Component
|
||||
import space.nstart.pcp.routepassportconsumer.config.RoutePassportOutboxProperties
|
||||
import space.nstart.pcp.routepassportconsumer.entity.RoutePassportOutboxEntity
|
||||
import space.nstart.pcp.routepassportconsumer.entity.RoutePassportOutboxStatus
|
||||
|
||||
/**
|
||||
* Scheduled worker that publishes already persisted route passport outbox rows.
|
||||
*/
|
||||
@Component
|
||||
class RoutePassportOutboxWorker(
|
||||
private val routePassportOutboxService: RoutePassportOutboxService,
|
||||
private val routePassportKafkaPublisherService: RoutePassportKafkaPublisherService,
|
||||
private val outboxProperties: RoutePassportOutboxProperties,
|
||||
) {
|
||||
private val log = LoggerFactory.getLogger(this::class.java)
|
||||
|
||||
@Scheduled(fixedDelayString = "\${app.route-passport.outbox.fixed-delay-ms:1000}")
|
||||
fun publishDueOutboxRows() {
|
||||
if (!outboxProperties.enabled) {
|
||||
return
|
||||
}
|
||||
|
||||
val outboxRows = routePassportOutboxService.claimDueBatch()
|
||||
outboxRows.forEach { outbox ->
|
||||
logClaimed(outbox)
|
||||
}
|
||||
|
||||
outboxRows.forEach { outbox ->
|
||||
try {
|
||||
publishOne(outbox)
|
||||
} catch (exception: Exception) {
|
||||
log.error(
|
||||
"Kafka outbox row processing failed: outboxId={}, routeId={}, eventType={}, topic={}, attempts={}, status={}, exceptionClass={}, exceptionMessage={}",
|
||||
outbox.id,
|
||||
outbox.routeId,
|
||||
outbox.eventType,
|
||||
outbox.topic,
|
||||
outbox.attempts,
|
||||
outbox.status,
|
||||
exception::class.java.name,
|
||||
exception.message,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun publishOne(outbox: RoutePassportOutboxEntity) {
|
||||
try {
|
||||
routePassportKafkaPublisherService.publish(outbox)
|
||||
routePassportOutboxService.markPublished(outbox.id)
|
||||
|
||||
log.info(
|
||||
"Kafka outbox row marked published: outboxId={}, routeId={}, eventType={}, topic={}, attempts={}, status={}",
|
||||
outbox.id,
|
||||
outbox.routeId,
|
||||
outbox.eventType,
|
||||
outbox.topic,
|
||||
outbox.attempts,
|
||||
RoutePassportOutboxStatus.PUBLISHED,
|
||||
)
|
||||
} catch (exception: Exception) {
|
||||
routePassportOutboxService.markRetryOrFailed(outbox.id, exception)
|
||||
|
||||
val attemptsAfterFailure = outbox.attempts + 1
|
||||
val statusAfterFailure = if (attemptsAfterFailure >= outbox.maxAttempts) {
|
||||
RoutePassportOutboxStatus.FAILED
|
||||
} else {
|
||||
RoutePassportOutboxStatus.PENDING
|
||||
}
|
||||
|
||||
log.warn(
|
||||
"Kafka outbox row publish failed: outboxId={}, routeId={}, eventType={}, topic={}, attempts={}, status={}, exceptionClass={}, exceptionMessage={}",
|
||||
outbox.id,
|
||||
outbox.routeId,
|
||||
outbox.eventType,
|
||||
outbox.topic,
|
||||
attemptsAfterFailure,
|
||||
statusAfterFailure,
|
||||
exception::class.java.name,
|
||||
exception.message,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun logClaimed(outbox: RoutePassportOutboxEntity) {
|
||||
log.info(
|
||||
"Kafka outbox row claimed: outboxId={}, routeId={}, eventType={}, topic={}, attempts={}, status={}",
|
||||
outbox.id,
|
||||
outbox.routeId,
|
||||
outbox.eventType,
|
||||
outbox.topic,
|
||||
outbox.attempts,
|
||||
outbox.status,
|
||||
)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user