[SYNTH-15675] Report timed out retries (#1440)

* [SYNTH-15675] Improve handling of non-final results

* Update fixtures

* Improve attempt text

* Improve test coverage + refactor

* Add jsdoc comment to `isResidualResult`

* Fix lint

src/commands/synthetics/__tests__/batch.test.ts

@@ -0,0 +1,140 @@
+import {getResultsToReport, reportReceivedResults} from '../batch'
+import {Batch, ResultInBatch} from '../interfaces'
+
+import {
+ getFailedResultInBatch,
+ getInProgressResultInBatch,
+ getPassedResultInBatch,
+ getSkippedResultInBatch,
+ mockReporter,
+} from './fixtures'
+
+describe('getResultsToReport', () => {
+ test.each([false])('timed out retry - shouldContinuePolling=%s', (shouldContinuePolling: boolean) => {
+ const timedOutRetry: ResultInBatch = {
+ ...getFailedResultInBatch(),
+ retries: 0,
+ max_retries: 1,
+ timed_out: true, // Can only be true when the backend timed out the batch, i.e. `shouldContinuePolling` is false.
+ }
+
+ const batch: Batch = {
+ status: 'failed',
+ results: [timedOutRetry],
+ }
+
+ const resultsToReport = getResultsToReport(
+ shouldContinuePolling,
+ batch,
+ [],
+ new Set(['rid']),
+ new Set(),
+ new Set(),
+ mockReporter
+ )
+
+ expect(resultsToReport).toStrictEqual([timedOutRetry])
+ })
+
+ test.each([false])(
+ 'timed out retry never emitted before - shouldContinuePolling=%s',
+ (shouldContinuePolling: boolean) => {
+ const timedOutRetry: ResultInBatch = {
+ ...getFailedResultInBatch(),
+ retries: 0,
+ max_retries: 1,
+ timed_out: true, // Can only be true when the backend timed out the batch, i.e. `shouldContinuePolling` is false.
+ }
+
+ const batch: Batch = {
+ status: 'failed',
+ results: [timedOutRetry],
+ }
+
+ const resultsToReport = getResultsToReport(
+ shouldContinuePolling,
+ batch,
+ [timedOutRetry],
+ new Set(),
+ new Set(),
+ new Set(),
+ mockReporter
+ )
+
+ expect(resultsToReport).toStrictEqual([timedOutRetry])
+ }
+ )
+})
+
+describe('reportReceivedResults', () => {
+ test('skipped', () => {
+ const skippedResult = getSkippedResultInBatch()
+
+ const batch: Batch = {
+ status: 'failed',
+ results: [skippedResult],
+ }
+
+ const emittedResultIds = new Set<string>()
+ const receivedResults = reportReceivedResults(batch, emittedResultIds, mockReporter)
+
+ expect(receivedResults).toStrictEqual([skippedResult])
+ expect(emittedResultIds).toContain('skipped-0')
+ expect(mockReporter.resultReceived).toHaveBeenCalledWith(skippedResult)
+ })
+
+ test('final', () => {
+ const result = getPassedResultInBatch()
+
+ const batch: Batch = {
+ status: 'passed',
+ results: [result],
+ }
+
+ const emittedResultIds = new Set<string>()
+ const receivedResults = reportReceivedResults(batch, emittedResultIds, mockReporter)
+
+ expect(receivedResults).toStrictEqual([result])
+ expect(emittedResultIds).toContain('rid')
+ expect(mockReporter.resultReceived).toHaveBeenCalledWith(result)
+ })
+
+ test('non final', () => {
+ const result: ResultInBatch = {
+ ...getInProgressResultInBatch(),
+ retries: 0,
+ max_retries: 1,
+ }
+
+ const batch: Batch = {
+ status: 'in_progress',
+ results: [result],
+ }
+
+ const emittedResultIds = new Set<string>()
+ const receivedResults = reportReceivedResults(batch, emittedResultIds, mockReporter)
+
+ expect(receivedResults).toStrictEqual([result])
+ expect(emittedResultIds).toContain('rid')
+ expect(mockReporter.resultReceived).toHaveBeenCalledWith(result)
+ })
+
+ test('timed out', () => {
+ const timedOut: ResultInBatch = {
+ ...getFailedResultInBatch(),
+ timed_out: true,
+ }
+
+ const batch: Batch = {
+ status: 'failed',
+ results: [timedOut],
+ }
+
+ const emittedResultIds = new Set<string>()
+ const receivedResults = reportReceivedResults(batch, emittedResultIds, mockReporter)
+
+ expect(receivedResults).toStrictEqual([timedOut])
+ expect(emittedResultIds).toContain('rid')
+ expect(mockReporter.resultReceived).toHaveBeenCalledWith(timedOut)
+ })
+})

src/commands/synthetics/__tests__/fixtures.ts

@@ -195,6 +195,7 @@ const getBaseResult = (resultId: string, test: Test): Omit<BaseResult, 'result'>
  passed: true,
  resultId,
  retries: 0,
+ maxRetries: 0,
  test,
  timedOut: false,
  timestamp: 1,
@@ -239,6 +240,7 @@ export const getTimedOutBrowserResult = (): Result => ({
  },
  resultId: '1',
  retries: 0,
+ maxRetries: 0,
  test: getBrowserTest(),
  timedOut: true,
  timestamp: 1,
@@ -299,6 +301,7 @@ export const getFailedBrowserResult = (): Result => ({
  },
  resultId: '1',
  retries: 0,
+ maxRetries: 0,
  test: getBrowserTest(),
  timedOut: false,
  timestamp: 1,
@@ -522,6 +525,8 @@ export const getInProgressResultInBatch = (): BaseResultInBatch => {
  result_id: 'rid',
  // eslint-disable-next-line no-null/no-null
  retries: null,
+ // eslint-disable-next-line no-null/no-null
+ max_retries: null,
  status: 'in_progress',
  test_public_id: 'pid',
  // eslint-disable-next-line no-null/no-null
@@ -535,6 +540,8 @@ export const getSkippedResultInBatch = (): ResultInBatchSkippedBySelectiveRerun
  execution_rule: ExecutionRule.SKIPPED,
  // eslint-disable-next-line no-null/no-null
  retries: null,
+ // eslint-disable-next-line no-null/no-null
+ max_retries: null,
  status: 'skipped',
  selective_rerun: {
  decision: 'skip',