feat(outbox): attempt cap → DLQ + admin endpoint + 6 unit-тестов

После N (default 1000) неудачных попыток публикации событие маркируется dlq_at и исключается из polling. Без cap poller бесконечно ретраил мёртвые сообщения (orphan topic, ACL deny), забивая логи и lag-метрику. - 0011-outbox-dlq.xml: dlq_at column + перестроенный idx_outbox_unpublished (исключает DLQ) + idx_outbox_dlq для admin-листинга. - OutboxPoller: ordinis.outbox.attempt-max (default 1000), markDlq() при достижении cap, новый counter ordinis_outbox_dlq_total. - OutboxLagGauge: ordinis_outbox_dlq_size gauge — алерт на > 0. - OutboxEventRepository: countPending() (без DLQ), findByDlqAtIsNotNull(Pageable). - OutboxAdminController: GET /admin/outbox/{stats,dlq} для UI. - 6 unit-тестов через mock-maker-subclass (JDK 25 ломает Mockito inline).
2026-05-04 15:43:56 +03:00
parent f7efb23e09
commit e182b30c58
10 changed files with 391 additions and 9 deletions
@@ -8,8 +8,13 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.stereotype.Component;

 /**
- * Gauge {@code ordinis_outbox_pending_events} — сколько unpublished. Алертится
- * Prometheus'ом если растёт (poller не успевает или Kafka недоступна).
+ * Gauges:
+ * <ul>
+ *   <li>{@code ordinis_outbox_pending_events} — unpublished без DLQ. Растёт ⇒
+ *       poller не успевает либо Kafka недоступна.</li>
+ *   <li>{@code ordinis_outbox_dlq_size} — события в DLQ. Любое значение &gt; 0 ⇒
+ *       требуется ручной разбор (orphan topic, ACL, broken payload).</li>
+ * </ul>
 */
@Component
@ConditionalOnProperty(name = "ordinis.outbox.enabled", havingValue = "true", matchIfMissing = false)
@@ -26,8 +31,13 @@ public class OutboxLagGauge {
  @PostConstruct
  void register() {
    Gauge.builder("ordinis_outbox_pending_events", repository,
-            r -> (double) r.countByPublishedAtIsNull())
-        .description("Количество outbox events ожидающих публикацию в Kafka")
+            r -> (double) r.countPending())
+        .description("Количество outbox events ожидающих публикацию (без DLQ)")
+        .register(meterRegistry);
+
+    Gauge.builder("ordinis_outbox_dlq_size", repository,
+            r -> (double) r.countByDlqAtIsNotNull())
+        .description("Количество outbox events в DLQ (требуют ручного разбора)")
        .register(meterRegistry);
  }
 }
@@ -25,6 +25,11 @@ import java.util.concurrent.TimeoutException;
 * @Scheduled poller для outbox. Sweep каждые {@code ordinis.outbox.poll-interval-ms},
 * batch size {@code ordinis.outbox.batch-size}. Активируется только если
 * {@code ordinis.outbox.enabled=true} (отключаем в read-api / projection-writer).
+ *
+ * <p>После {@code ordinis.outbox.attempt-max} (default 1000) неудачных попыток
+ * событие маркируется {@code dlq_at} и исключается из дальнейшего polling. Без
+ * cap poller бесконечно ретраит мёртвые сообщения (orphan topic, ACL deny),
+ * забивая лог и lag-метрику.
 */
@Component
@ConditionalOnProperty(name = "ordinis.outbox.enabled", havingValue = "true", matchIfMissing = false)
@@ -37,6 +42,7 @@ public class OutboxPoller {
  private final com.fasterxml.jackson.databind.ObjectMapper objectMapper;
  private final Counter publishedCounter;
  private final Counter failedCounter;
+  private final Counter dlqCounter;
  private final Timer publishTimer;

  @Value("${ordinis.outbox.batch-size:100}")
@@ -45,6 +51,9 @@ public class OutboxPoller {
  @Value("${ordinis.outbox.send-timeout-ms:10000}")
  private long sendTimeoutMs;

+  @Value("${ordinis.outbox.attempt-max:1000}")
+  private int attemptMax;
+
  public OutboxPoller(
      OutboxEventRepository repository,
      KafkaTemplate<String, String> kafkaTemplate,
@@ -59,6 +68,9 @@ public class OutboxPoller {
    this.failedCounter = Counter.builder("ordinis_outbox_publish_errors_total")
        .description("Количество ошибок публикации в Kafka")
        .register(meterRegistry);
+    this.dlqCounter = Counter.builder("ordinis_outbox_dlq_total")
+        .description("Количество событий перенесённых в DLQ (attempt cap)")
+        .register(meterRegistry);
    this.publishTimer = Timer.builder("ordinis_kafka_publish_duration_seconds")
        .description("Время публикации одного события в Kafka")
        .register(meterRegistry);
@@ -103,10 +115,21 @@ public class OutboxPoller {
  }

  private void markFailed(OutboxEvent event, String error) {
-    log.warn("Failed to publish outbox event id={} topic={}: {}",
-        event.getId(), event.getKafkaTopic(), error);
    event.markFailed(error);
-    repository.save(event);
    failedCounter.increment();
+
+    if (event.getAttemptCount() != null && event.getAttemptCount() >= attemptMax) {
+      event.markDlq(error);
+      dlqCounter.increment();
+      log.error(
+          "Outbox event id={} topic={} → DLQ after {} attempts. Last error: {}",
+          event.getId(), event.getKafkaTopic(), event.getAttemptCount(), error);
+    } else {
+      log.warn(
+          "Failed to publish outbox event id={} topic={} attempt={}/{}: {}",
+          event.getId(), event.getKafkaTopic(),
+          event.getAttemptCount(), attemptMax, error);
+    }
+    repository.save(event);
  }
 }
@@ -0,0 +1,170 @@
+package cloud.nstart.terravault.ordinis.outbox;
+
+import cloud.nstart.terravault.ordinis.domain.DataScope;
+import cloud.nstart.terravault.ordinis.domain.outbox.OutboxEvent;
+import cloud.nstart.terravault.ordinis.domain.outbox.OutboxEventRepository;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.TopicPartition;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.data.domain.Pageable;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.kafka.support.SendResult;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeoutException;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class OutboxPollerTest {
+
+  private OutboxEventRepository repo;
+  private KafkaTemplate<String, String> kafka;
+  private OutboxPoller poller;
+
+  @BeforeEach
+  void setUp() {
+    repo = mock(OutboxEventRepository.class);
+    kafka = mock(KafkaTemplate.class);
+    poller = new OutboxPoller(repo, kafka, new ObjectMapper(), new SimpleMeterRegistry());
+    ReflectionTestUtils.setField(poller, "batchSize", 100);
+    ReflectionTestUtils.setField(poller, "sendTimeoutMs", 1_000L);
+    ReflectionTestUtils.setField(poller, "attemptMax", 3);
+  }
+
+  private OutboxEvent newEvent() {
+    return new OutboxEvent(
+        "DICT_RECORD_CREATED",
+        "DictionaryRecord",
+        "spacecraft:RESURS-P-3",
+        DataScope.PUBLIC,
+        JsonNodeFactory.instance.objectNode().put("foo", "bar"),
+        "ordinis.cuod.events.public",
+        "spacecraft:RESURS-P-3");
+  }
+
+  @Test
+  void empty_batch_does_nothing() {
+    when(repo.findUnpublished(any(Pageable.class))).thenReturn(List.of());
+
+    poller.poll();
+
+    verify(kafka, never()).send(anyString(), anyString(), anyString());
+    verify(repo, never()).save(any());
+  }
+
+  @Test
+  void successful_publish_marks_published() {
+    OutboxEvent event = newEvent();
+    when(repo.findUnpublished(any(Pageable.class))).thenReturn(List.of(event));
+
+    SendResult<String, String> result = mock(SendResult.class);
+    when(result.getRecordMetadata()).thenReturn(
+        new RecordMetadata(new TopicPartition("ordinis.cuod.events.public", 0), 0L, 0, 0L, 0, 0));
+    when(kafka.send(eq("ordinis.cuod.events.public"), anyString(), anyString()))
+        .thenReturn(CompletableFuture.completedFuture(result));
+
+    poller.poll();
+
+    assertThat(event.getPublishedAt()).isNotNull();
+    assertThat(event.getDlqAt()).isNull();
+    verify(repo).save(event);
+  }
+
+  @Test
+  void failure_below_cap_marks_failed_not_dlq() {
+    OutboxEvent event = newEvent();
+    when(repo.findUnpublished(any(Pageable.class))).thenReturn(List.of(event));
+
+    CompletableFuture<SendResult<String, String>> failed = new CompletableFuture<>();
+    failed.completeExceptionally(new TimeoutException("kafka down"));
+    when(kafka.send(anyString(), anyString(), anyString())).thenReturn(failed);
+
+    poller.poll();
+
+    assertThat(event.getAttemptCount()).isEqualTo(1);
+    assertThat(event.getDlqAt()).isNull();
+    assertThat(event.getLastError()).contains("kafka down");
+    verify(repo).save(event);
+  }
+
+  @Test
+  void failure_at_cap_marks_dlq() {
+    OutboxEvent event = newEvent();
+    // Симулируем что 2 предыдущих попытки уже были.
+    event.markFailed("prev 1");
+    event.markFailed("prev 2");
+    assertThat(event.getAttemptCount()).isEqualTo(2);
+
+    when(repo.findUnpublished(any(Pageable.class))).thenReturn(List.of(event));
+
+    CompletableFuture<SendResult<String, String>> failed = new CompletableFuture<>();
+    failed.completeExceptionally(new TimeoutException("still down"));
+    when(kafka.send(anyString(), anyString(), anyString())).thenReturn(failed);
+
+    poller.poll();
+
+    assertThat(event.getAttemptCount()).isEqualTo(3);
+    assertThat(event.getDlqAt()).isNotNull();
+    assertThat(event.isDlq()).isTrue();
+    assertThat(event.getLastError()).contains("still down");
+    verify(repo).save(event);
+  }
+
+  @Test
+  void failure_above_cap_stays_dlq_not_re_marked() {
+    OutboxEvent event = newEvent();
+    for (int i = 0; i < 3; i++) event.markFailed("err " + i);
+    event.markDlq("first dlq");
+    java.time.OffsetDateTime firstDlqAt = event.getDlqAt();
+
+    when(repo.findUnpublished(any(Pageable.class))).thenReturn(List.of());
+
+    poller.poll();
+
+    // Poller не должен подобрать DLQ событие — findUnpublished фильтрует dlq_at IS NULL.
+    assertThat(event.getDlqAt()).isEqualTo(firstDlqAt);
+  }
+
+  @Test
+  void mixed_batch_publishes_some_dlqs_others() {
+    OutboxEvent ok = newEvent();
+    OutboxEvent doomed = newEvent();
+    doomed.markFailed("p1");
+    doomed.markFailed("p2");
+
+    when(repo.findUnpublished(any(Pageable.class))).thenReturn(List.of(ok, doomed));
+
+    SendResult<String, String> result = mock(SendResult.class);
+    when(result.getRecordMetadata()).thenReturn(
+        new RecordMetadata(new TopicPartition("ordinis.cuod.events.public", 0), 0L, 0, 0L, 0, 0));
+    CompletableFuture<SendResult<String, String>> okFut = CompletableFuture.completedFuture(result);
+    CompletableFuture<SendResult<String, String>> badFut = new CompletableFuture<>();
+    badFut.completeExceptionally(new TimeoutException("doomed"));
+
+    // Первый event получает success, второй — failure.
+    when(kafka.send(anyString(), anyString(), anyString()))
+        .thenReturn(okFut)
+        .thenReturn(badFut);
+
+    poller.poll();
+
+    assertThat(ok.getPublishedAt()).isNotNull();
+    assertThat(ok.getDlqAt()).isNull();
+    assertThat(doomed.getDlqAt()).isNotNull();
+    verify(repo, times(2)).save(any());
+  }
+}
@@ -0,0 +1 @@
+mock-maker-subclass