From 73aba32cfe626e9df4267c2bb565f0b35e92ebef Mon Sep 17 00:00:00 2001 From: Stanislav Deviatykh Date: Fri, 5 Jun 2026 19:05:05 +0200 Subject: [PATCH 1/9] feat(graphql): block retry if connection not established --- .../graphql-blocked-websocket.spec.ts | 96 +++++++++++++++++++ .../lib/services/graphql/graphql.service.ts | 43 +++++++-- 2 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 packages/javascript-api/src/lib/services/graphql/__tests__/graphql-blocked-websocket.spec.ts diff --git a/packages/javascript-api/src/lib/services/graphql/__tests__/graphql-blocked-websocket.spec.ts b/packages/javascript-api/src/lib/services/graphql/__tests__/graphql-blocked-websocket.spec.ts new file mode 100644 index 00000000..0e3af5f4 --- /dev/null +++ b/packages/javascript-api/src/lib/services/graphql/__tests__/graphql-blocked-websocket.spec.ts @@ -0,0 +1,96 @@ +import { WebSocket } from 'mock-socket'; + +import { GraphQLSubscriptionsFixture } from '../__fixtures__/graphql-subscriptions-fixture'; +import * as backoff from '../../../util/randomized-exponential-backoff/randomized-exponential-backoff'; + +jest.mock('isomorphic-ws', () => WebSocket); +jest.mock('../../../util/sleep-ms/sleep-ms', () => ({ + sleepMs: () => new Promise((resolve) => setTimeout(resolve, 4)), +})); + +// Mirrors the constants in graphql.service.ts (not exported). +const MAX_FAILED_HANDSHAKES = 5; +const BLOCKED_RETRY_INTERVAL_MS = 5 * 60 * 1000; + +describe('GraphQL blocked-WebSocket back-off', () => { + let fixture: GraphQLSubscriptionsFixture; + + beforeEach(() => { + fixture = new GraphQLSubscriptionsFixture(); + }); + + afterEach(async () => { + await fixture.cleanup(); + }); + + it('should back off to a slow interval after repeated handshakes fail before connecting', async () => { + const service = fixture.graphqlService as any; + const backoffSpy = jest.spyOn( + backoff, + 'calculateRandomizedExponentialBackoffTime', + ); + const infoSpy = jest.spyOn(service.logger, 'info'); + + const sub = fixture.triggerSubscription(); + + // Each cycle: connect, never ACK, close before the connection is established. + for (let i = 0; i < MAX_FAILED_HANDSHAKES; i++) { + await fixture.waitForConnection(); + await fixture.getNextMessage(); // consume connection_init + await fixture.closeWithCode(1001); + fixture.openServer(); + } + await new Promise((resolve) => setTimeout(resolve, 20)); + + expect(service.consecutiveFailedHandshakes).toBeGreaterThanOrEqual( + MAX_FAILED_HANDSHAKES, + ); + // Fast exponential backoff is used only for the first (MAX - 1) reconnects; + // once the cap is hit, the flat slow interval is used instead. + expect(backoffSpy).toHaveBeenCalledTimes(MAX_FAILED_HANDSHAKES - 1); + expect(infoSpy).toHaveBeenCalledWith( + `Reconnect socket in ${BLOCKED_RETRY_INTERVAL_MS}ms`, + ); + + sub.unsubscribe(); + }); + + it('should not count a close that happens after a successful connection', async () => { + const service = fixture.graphqlService as any; + + const sub = fixture.triggerSubscription(); + await fixture.handleConnectionInit(); // connection_ack -> CONNECTED + await fixture.consumeSubscribeMessage(); + + expect(service.consecutiveFailedHandshakes).toBe(0); + + // A drop after the connection was established is a normal reconnect. + await fixture.closeWithCode(1001); + expect(service.consecutiveFailedHandshakes).toBe(0); + + fixture.openServer(); + await fixture.handleConnectionInit(); + sub.unsubscribe(); + }); + + it('should reset the counter when the connection is established', async () => { + const service = fixture.graphqlService as any; + + const sub = fixture.triggerSubscription(); + + for (let i = 0; i < 2; i++) { + await fixture.waitForConnection(); + await fixture.getNextMessage(); + await fixture.closeWithCode(1001); + fixture.openServer(); + } + expect(service.consecutiveFailedHandshakes).toBe(2); + + // A successful connection_ack resets the counter back to fast reconnects. + await fixture.handleConnectionInit(); + await fixture.consumeSubscribeMessage(); + expect(service.consecutiveFailedHandshakes).toBe(0); + + sub.unsubscribe(); + }); +}); diff --git a/packages/javascript-api/src/lib/services/graphql/graphql.service.ts b/packages/javascript-api/src/lib/services/graphql/graphql.service.ts index a45c71e5..2b028367 100644 --- a/packages/javascript-api/src/lib/services/graphql/graphql.service.ts +++ b/packages/javascript-api/src/lib/services/graphql/graphql.service.ts @@ -98,6 +98,12 @@ const PING_PONG_INTERVAL_IN_MS = 20_000; // https://www.w3.org/TR/websockets/#concept-websocket-close-fail const CLIENT_SIDE_CLOSE_EVENT = 1000; +// Once the WebSocket has failed to establish this many times in a row, the reconnect +// loop backs off to a slow interval instead of minting a new temporary API key on +// every fast retry. A successful connection resets the counter. +const MAX_FAILED_HANDSHAKES = 5; +const BLOCKED_RETRY_INTERVAL_MS = 5 * 60 * 1000; + /** * A service that lets the user query Qminder API via GraphQL statements. * Queries and subscriptions are supported. There is no support for mutations. @@ -192,6 +198,10 @@ export class GraphqlService { private connectionAttemptsCount = 0; + // Consecutive WebSocket closes that happened before the connection was ever + // established. Reset to 0 on a successful GQL_CONNECTION_ACK. + private consecutiveFailedHandshakes = 0; + constructor() { this.setServer('api.qminder.com'); @@ -446,14 +456,32 @@ export class GraphqlService { reason: event.reason, }); + // Capture this before the status is overwritten below: a close while still + // CONNECTING means the connection never established. + const closedBeforeEstablished = + this.connectionStatus === ConnectionStatus.CONNECTING; + this.setConnectionStatus(ConnectionStatus.DISCONNECTED); this.socket = null; this.clearPingMonitoring(); - if (this.shouldRetry(event)) { - const timer = calculateRandomizedExponentialBackoffTime( - this.connectionAttemptsCount, + if (closedBeforeEstablished) { + this.consecutiveFailedHandshakes++; + this.logger.error( + `Received socket close event before a connection was established! Close code: ${event.code}`, ); + } + + if (this.shouldRetry(event)) { + // After repeated failures to establish, back off to a slow interval so we + // stop creating a temporary API key on every retry. Recovers on its own: + // a successful connection resets consecutiveFailedHandshakes. + const timer = + this.consecutiveFailedHandshakes >= MAX_FAILED_HANDSHAKES + ? BLOCKED_RETRY_INTERVAL_MS + : calculateRandomizedExponentialBackoffTime( + this.connectionAttemptsCount, + ); this.logger.info(`Reconnect socket in ${timer.toFixed(0)}ms`); @@ -466,12 +494,6 @@ export class GraphqlService { this.logger.error('Failed to reconnect socket: ', error); }); } - - if (this.connectionStatus === ConnectionStatus.CONNECTING) { - this.logger.error( - `Received socket close event before a connection was established! Close code: ${event.code}`, - ); - } }; this.socket.onerror = () => { @@ -498,6 +520,9 @@ export class GraphqlService { this.retryableErroredSubscriptionsRetryCount = 0; this.retryableErroredSubscriptionsAction$.next({ type: 'clear' }); + // Connection established — clear the blocked back-off. + this.consecutiveFailedHandshakes = 0; + this.setConnectionStatus(ConnectionStatus.CONNECTED); this.logger.info('Connected to websocket'); this.startConnectionMonitoring(); From a400afda83126aa21e343dd22877833210c5bbf9 Mon Sep 17 00:00:00 2001 From: Stanislav Deviatykh Date: Mon, 15 Jun 2026 09:48:53 +0200 Subject: [PATCH 2/9] polish spec --- .../graphql/__tests__/graphql-blocked-websocket.spec.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/javascript-api/src/lib/services/graphql/__tests__/graphql-blocked-websocket.spec.ts b/packages/javascript-api/src/lib/services/graphql/__tests__/graphql-blocked-websocket.spec.ts index 0e3af5f4..b2ce382a 100644 --- a/packages/javascript-api/src/lib/services/graphql/__tests__/graphql-blocked-websocket.spec.ts +++ b/packages/javascript-api/src/lib/services/graphql/__tests__/graphql-blocked-websocket.spec.ts @@ -36,11 +36,10 @@ describe('GraphQL blocked-WebSocket back-off', () => { // Each cycle: connect, never ACK, close before the connection is established. for (let i = 0; i < MAX_FAILED_HANDSHAKES; i++) { await fixture.waitForConnection(); - await fixture.getNextMessage(); // consume connection_init + await fixture.consumeInitMessage(); await fixture.closeWithCode(1001); fixture.openServer(); } - await new Promise((resolve) => setTimeout(resolve, 20)); expect(service.consecutiveFailedHandshakes).toBeGreaterThanOrEqual( MAX_FAILED_HANDSHAKES, From 19d8cefa6bd4c7133aff91fdbc338d770c490c43 Mon Sep 17 00:00:00 2001 From: Stanislav Deviatykh Date: Mon, 15 Jun 2026 09:49:00 +0200 Subject: [PATCH 3/9] polish graphql service --- .../lib/services/graphql/graphql.service.ts | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/packages/javascript-api/src/lib/services/graphql/graphql.service.ts b/packages/javascript-api/src/lib/services/graphql/graphql.service.ts index 2b028367..ad710ad9 100644 --- a/packages/javascript-api/src/lib/services/graphql/graphql.service.ts +++ b/packages/javascript-api/src/lib/services/graphql/graphql.service.ts @@ -468,7 +468,8 @@ export class GraphqlService { if (closedBeforeEstablished) { this.consecutiveFailedHandshakes++; this.logger.error( - `Received socket close event before a connection was established! Close code: ${event.code}`, + `Received socket close event before a connection was established! ` + + `Close code: ${event.code} (consecutive failed handshakes: ${this.consecutiveFailedHandshakes})`, ); } @@ -476,12 +477,21 @@ export class GraphqlService { // After repeated failures to establish, back off to a slow interval so we // stop creating a temporary API key on every retry. Recovers on its own: // a successful connection resets consecutiveFailedHandshakes. - const timer = - this.consecutiveFailedHandshakes >= MAX_FAILED_HANDSHAKES - ? BLOCKED_RETRY_INTERVAL_MS - : calculateRandomizedExponentialBackoffTime( - this.connectionAttemptsCount, - ); + const isBlocked = + this.consecutiveFailedHandshakes >= MAX_FAILED_HANDSHAKES; + + if (isBlocked) { + this.logger.warn( + `Handshake failed ${this.consecutiveFailedHandshakes} times in a row; ` + + `backing off to ${BLOCKED_RETRY_INTERVAL_MS}ms to stop creating temporary API keys on every retry.`, + ); + } + + const timer = isBlocked + ? BLOCKED_RETRY_INTERVAL_MS + : calculateRandomizedExponentialBackoffTime( + this.connectionAttemptsCount, + ); this.logger.info(`Reconnect socket in ${timer.toFixed(0)}ms`); From 0795f81b97148af37b565c7d781e3046391c53e0 Mon Sep 17 00:00:00 2001 From: Stanislav Deviatykh Date: Mon, 15 Jun 2026 10:58:47 +0200 Subject: [PATCH 4/9] refactor log --- .../javascript-api/src/lib/services/graphql/graphql.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/javascript-api/src/lib/services/graphql/graphql.service.ts b/packages/javascript-api/src/lib/services/graphql/graphql.service.ts index ad710ad9..45a10bdb 100644 --- a/packages/javascript-api/src/lib/services/graphql/graphql.service.ts +++ b/packages/javascript-api/src/lib/services/graphql/graphql.service.ts @@ -483,7 +483,7 @@ export class GraphqlService { if (isBlocked) { this.logger.warn( `Handshake failed ${this.consecutiveFailedHandshakes} times in a row; ` + - `backing off to ${BLOCKED_RETRY_INTERVAL_MS}ms to stop creating temporary API keys on every retry.`, + `slowing down reconnect attempts to ${BLOCKED_RETRY_INTERVAL_MS}ms.`, ); } From 0bacdaf40269ac9ae9b76957b9c7d3534925afb5 Mon Sep 17 00:00:00 2001 From: Stanislav Deviatykh Date: Mon, 15 Jun 2026 10:59:14 +0200 Subject: [PATCH 5/9] add teardownService --- .../graphql-subscriptions-fixture.ts | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts b/packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts index 42969500..d43aa59f 100644 --- a/packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts +++ b/packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts @@ -123,7 +123,27 @@ export class GraphQLSubscriptionsFixture { } async cleanup() { + this.teardownService(); WS.clean(); await this.server.closed; } + + private teardownService() { + const service = this.graphqlService as any; + + service.openSocket = () => Promise.resolve(); + + if (service.socket) { + service.socket.onopen = null; + service.socket.onmessage = null; + service.socket.onerror = null; + service.socket.onclose = null; + if (typeof service.socket.close === 'function') { + service.socket.close(); + } + service.socket = null; + } + + service.clearPingMonitoring(); + } } From 52006b545b8808454b6ab8af951d7b9b32649580 Mon Sep 17 00:00:00 2001 From: Stanislav Deviatykh Date: Mon, 15 Jun 2026 14:07:05 +0200 Subject: [PATCH 6/9] Update packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts Co-authored-by: Rando Luik <47411930+raluik@users.noreply.github.com> --- .../graphql/__fixtures__/graphql-subscriptions-fixture.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts b/packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts index d43aa59f..97480ed6 100644 --- a/packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts +++ b/packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts @@ -122,7 +122,7 @@ export class GraphQLSubscriptionsFixture { await this.server.nextMessage; } - async cleanup() { + async cleanup(): Promise { this.teardownService(); WS.clean(); await this.server.closed; From 33627b4a84b25ff8b98d7fb27871f692ed5a7668 Mon Sep 17 00:00:00 2001 From: Stanislav Deviatykh Date: Mon, 15 Jun 2026 14:07:13 +0200 Subject: [PATCH 7/9] Update packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts Co-authored-by: Rando Luik <47411930+raluik@users.noreply.github.com> --- .../graphql-subscriptions-fixture.ts | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts b/packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts index 97480ed6..d2167cc1 100644 --- a/packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts +++ b/packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts @@ -128,7 +128,25 @@ export class GraphQLSubscriptionsFixture { await this.server.closed; } - private teardownService() { + private tearDownService(): void { + this.graphqlService['openSocket'] = () => Promise.resolve(); + + const socket = this.graphqlService['socket']; + if (socket) { + socket.onopen = null; + socket.onmessage = null; + socket.onerror = null; + socket.onclose = null; + + if (typeof socket.close === 'function') { + socket.close(); + } + + this.graphqlService['socket'] = null; + } + + this.graphqlService['clearPingMonitoring'](); + } const service = this.graphqlService as any; service.openSocket = () => Promise.resolve(); From 53bfaa4cae6c1c651c09f6544d9ddc8cfa4e156d Mon Sep 17 00:00:00 2001 From: Stanislav Deviatykh Date: Mon, 15 Jun 2026 14:07:19 +0200 Subject: [PATCH 8/9] Update packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts Co-authored-by: Rando Luik <47411930+raluik@users.noreply.github.com> --- .../graphql/__fixtures__/graphql-subscriptions-fixture.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts b/packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts index d2167cc1..8812300c 100644 --- a/packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts +++ b/packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts @@ -123,7 +123,7 @@ export class GraphQLSubscriptionsFixture { } async cleanup(): Promise { - this.teardownService(); + this.tearDownService(); WS.clean(); await this.server.closed; } From b41fa116d1c035deaedc5c6a6fbad70e17e1da65 Mon Sep 17 00:00:00 2001 From: Stanislav Deviatykh Date: Mon, 15 Jun 2026 14:09:37 +0200 Subject: [PATCH 9/9] fix --- .../graphql-subscriptions-fixture.ts | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts b/packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts index 8812300c..30459eb6 100644 --- a/packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts +++ b/packages/javascript-api/src/lib/services/graphql/__fixtures__/graphql-subscriptions-fixture.ts @@ -147,21 +147,4 @@ export class GraphQLSubscriptionsFixture { this.graphqlService['clearPingMonitoring'](); } - const service = this.graphqlService as any; - - service.openSocket = () => Promise.resolve(); - - if (service.socket) { - service.socket.onopen = null; - service.socket.onmessage = null; - service.socket.onerror = null; - service.socket.onclose = null; - if (typeof service.socket.close === 'function') { - service.socket.close(); - } - service.socket = null; - } - - service.clearPingMonitoring(); - } }