From 18a7b233f0bac1329f2ee4526a035f38c353748c Mon Sep 17 00:00:00 2001 From: Bhagya Amarasinghe Date: Fri, 13 Feb 2026 02:31:21 +0530 Subject: [PATCH] fix: distributed lock for license fetch when Redis cache is cold (#7225) Co-authored-by: Cursor --- .../ee/license-check/lib/license.test.ts | 290 +++++++++++++++++- .../modules/ee/license-check/lib/license.ts | 86 ++++-- packages/cache/src/cache-keys.ts | 1 + 3 files changed, 355 insertions(+), 22 deletions(-) diff --git a/apps/web/modules/ee/license-check/lib/license.test.ts b/apps/web/modules/ee/license-check/lib/license.test.ts index 90b814550528..da335d0df033 100644 --- a/apps/web/modules/ee/license-check/lib/license.test.ts +++ b/apps/web/modules/ee/license-check/lib/license.test.ts @@ -24,6 +24,7 @@ const mockCache = { set: vi.fn(), del: vi.fn(), exists: vi.fn(), + tryLock: vi.fn(), withCache: vi.fn(), getRedisClient: vi.fn(), }; @@ -38,6 +39,7 @@ vi.mock("@formbricks/cache", () => ({ license: { status: (identifier: string) => `fb:license:${identifier}:status`, previous_result: (identifier: string) => `fb:license:${identifier}:previous_result`, + fetch_lock: (identifier: string) => `fb:license:${identifier}:fetch_lock`, }, custom: (namespace: string, identifier: string, subResource?: string) => { const base = `fb:${namespace}:${identifier}`; @@ -99,6 +101,7 @@ describe("License Core Logic", () => { mockCache.set.mockReset(); mockCache.del.mockReset(); mockCache.exists.mockReset(); + mockCache.tryLock.mockReset(); mockCache.withCache.mockReset(); mockLogger.error.mockReset(); mockLogger.warn.mockReset(); @@ -106,9 +109,10 @@ describe("License Core Logic", () => { mockLogger.debug.mockReset(); // Set up default mock implementations for Result types - // fetchLicense uses get + exists; getPreviousResult uses get with :previous_result key + // fetchLicense uses get with TCachedFetchResult wrapper + distributed lock; getPreviousResult uses get with :previous_result key mockCache.get.mockResolvedValue({ ok: true, data: null }); mockCache.exists.mockResolvedValue({ ok: true, data: false }); // default: cache miss + mockCache.tryLock.mockResolvedValue({ ok: true, data: true }); // default: lock acquired mockCache.set.mockResolvedValue({ ok: true }); vi.mocked(prisma.response.count).mockResolvedValue(100); @@ -180,6 +184,8 @@ describe("License Core Logic", () => { const license = await getEnterpriseLicense(); expect(license).toEqual(expectedActiveLicenseState); expect(mockCache.get).toHaveBeenCalledWith(expect.stringContaining("fb:license:")); + // Should have checked cache but NOT acquired lock or called fetch + expect(mockCache.tryLock).not.toHaveBeenCalled(); expect(fetch).not.toHaveBeenCalled(); }); @@ -197,6 +203,9 @@ describe("License Core Logic", () => { expect(fetch).toHaveBeenCalledTimes(1); expect(mockCache.get).toHaveBeenCalledWith(expect.stringContaining("fb:license:")); + // Should have tried to acquire lock and set the cache + expect(mockCache.tryLock).toHaveBeenCalled(); + expect(mockCache.set).toHaveBeenCalled(); expect(license).toEqual(expectedActiveLicenseState); }); @@ -388,6 +397,7 @@ describe("License Core Logic", () => { expect(mockCache.get).not.toHaveBeenCalled(); expect(mockCache.set).not.toHaveBeenCalled(); expect(mockCache.exists).not.toHaveBeenCalled(); + expect(mockCache.tryLock).not.toHaveBeenCalled(); }); test("should handle fetch throwing an error and use grace period or return inactive", async () => { @@ -455,6 +465,280 @@ describe("License Core Logic", () => { status: "invalid_license" as const, }); }); + + test("should skip polling and fetch directly when Redis is unavailable (tryLock error)", async () => { + vi.resetModules(); + vi.doMock("@/lib/env", () => ({ + env: { + ENTERPRISE_LICENSE_KEY: "test-license-key", + ENVIRONMENT: "production", + VERCEL_URL: "some.vercel.url", + FORMBRICKS_COM_URL: "https://app.formbricks.com", + HTTPS_PROXY: undefined, + HTTP_PROXY: undefined, + }, + })); + + const { getEnterpriseLicense } = await import("./license"); + const fetch = (await import("node-fetch")).default as Mock; + + const mockLicense: TEnterpriseLicenseDetails = { + status: "active", + features: { + isMultiOrgEnabled: true, + contacts: true, + projects: 10, + whitelabel: true, + removeBranding: true, + twoFactorAuth: true, + sso: true, + saml: true, + spamProtection: true, + ai: false, + auditLogs: true, + multiLanguageSurveys: true, + accessControl: true, + quotas: true, + }, + }; + + // Redis is down: cache.get returns error, tryLock returns error + mockCache.get.mockResolvedValue({ ok: false, error: { code: "redis_connection_error" } }); + mockCache.tryLock.mockResolvedValue({ ok: false, error: { code: "redis_connection_error" } }); + + fetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ data: mockLicense }), + } as any); + + const startTime = Date.now(); + const license = await getEnterpriseLicense(); + const elapsed = Date.now() - startTime; + + // Should NOT have waited for polling — should complete quickly + expect(elapsed).toBeLessThan(5000); + expect(fetch).toHaveBeenCalledTimes(1); + expect(mockLogger.warn).toHaveBeenCalledWith( + "Redis unavailable during license fetch lock; skipping poll and fetching directly" + ); + expect(license).toEqual({ + active: true, + features: mockLicense.features, + lastChecked: expect.any(Date), + isPendingDowngrade: false, + fallbackLevel: "live" as const, + status: "active" as const, + }); + }); + + test("should poll and return cached value when another process holds the lock", async () => { + vi.resetModules(); + vi.doMock("@/lib/env", () => ({ + env: { + ENTERPRISE_LICENSE_KEY: "test-license-key", + ENVIRONMENT: "production", + VERCEL_URL: "some.vercel.url", + FORMBRICKS_COM_URL: "https://app.formbricks.com", + HTTPS_PROXY: undefined, + HTTP_PROXY: undefined, + }, + })); + + const { fetchLicense } = await import("./license"); + const fetch = (await import("node-fetch")).default as Mock; + + const mockLicense: TEnterpriseLicenseDetails = { + status: "active", + features: { + isMultiOrgEnabled: true, + contacts: true, + projects: 10, + whitelabel: true, + removeBranding: true, + twoFactorAuth: true, + sso: true, + saml: true, + spamProtection: true, + ai: false, + auditLogs: true, + multiLanguageSurveys: true, + accessControl: true, + quotas: true, + }, + }; + + // Lock held by another process (ok: true, data: false) + mockCache.tryLock.mockResolvedValue({ ok: true, data: false }); + + // First get returns cache miss, subsequent gets return the populated license + let getCalls = 0; + mockCache.get.mockImplementation(async (key: string) => { + if (key.includes(":status")) { + getCalls++; + if (getCalls <= 1) return { ok: true, data: null }; + return { ok: true, data: { value: mockLicense } }; + } + return { ok: true, data: null }; + }); + + const result = await fetchLicense(); + + expect(fetch).not.toHaveBeenCalled(); + expect(result).toEqual(mockLicense); + }); + + test("should fall back to direct fetch after poll timeout when lock is held", async () => { + vi.resetModules(); + vi.doMock("@/lib/env", () => ({ + env: { + ENTERPRISE_LICENSE_KEY: "test-license-key", + ENVIRONMENT: "production", + VERCEL_URL: "some.vercel.url", + FORMBRICKS_COM_URL: "https://app.formbricks.com", + HTTPS_PROXY: undefined, + HTTP_PROXY: undefined, + }, + })); + + const { fetchLicense } = await import("./license"); + const fetch = (await import("node-fetch")).default as Mock; + + const mockLicense: TEnterpriseLicenseDetails = { + status: "active", + features: { + isMultiOrgEnabled: true, + contacts: true, + projects: 10, + whitelabel: true, + removeBranding: true, + twoFactorAuth: true, + sso: true, + saml: true, + spamProtection: true, + ai: false, + auditLogs: true, + multiLanguageSurveys: true, + accessControl: true, + quotas: true, + }, + }; + + // Lock held, cache never gets populated + mockCache.tryLock.mockResolvedValue({ ok: true, data: false }); + mockCache.get.mockResolvedValue({ ok: true, data: null }); + mockCache.set.mockResolvedValue({ ok: true }); + + fetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ data: mockLicense }), + } as any); + + const result = await fetchLicense(); + + expect(fetch).toHaveBeenCalledTimes(1); + expect(mockLogger.warn).toHaveBeenCalledWith( + expect.objectContaining({ pollMs: expect.any(Number) }), + "License cache not populated by holder within poll window; fetching in this process" + ); + expect(result).toEqual(mockLicense); + }); + + test("should log warning and use short TTL when lock acquired but fetch returns null", async () => { + vi.resetModules(); + vi.doMock("@/lib/env", () => ({ + env: { + ENTERPRISE_LICENSE_KEY: "test-license-key", + ENVIRONMENT: "production", + VERCEL_URL: "some.vercel.url", + FORMBRICKS_COM_URL: "https://app.formbricks.com", + HTTPS_PROXY: undefined, + HTTP_PROXY: undefined, + }, + })); + + const { fetchLicense } = await import("./license"); + const fetch = (await import("node-fetch")).default as Mock; + + mockCache.get.mockResolvedValue({ ok: true, data: null }); + mockCache.tryLock.mockResolvedValue({ ok: true, data: true }); + mockCache.set.mockResolvedValue({ ok: true }); + + fetch.mockResolvedValueOnce({ ok: false, status: 500 } as any); + + const result = await fetchLicense(); + + expect(result).toBeNull(); + expect(mockLogger.warn).toHaveBeenCalledWith( + expect.objectContaining({ + ttlMinutes: expect.any(Number), + timestamp: expect.any(String), + }), + "License fetch failed, caching null result with short TTL for faster retry" + ); + expect(mockCache.set).toHaveBeenCalledWith( + expect.stringContaining("fb:license:"), + { value: null }, + 10 * 60 * 1000 + ); + }); + + test("should return null during build time (NEXT_PHASE = phase-production-build)", async () => { + vi.resetModules(); + vi.doMock("@/lib/env", () => ({ + env: { + ENTERPRISE_LICENSE_KEY: "test-license-key", + ENVIRONMENT: "production", + VERCEL_URL: "some.vercel.url", + FORMBRICKS_COM_URL: "https://app.formbricks.com", + HTTPS_PROXY: undefined, + HTTP_PROXY: undefined, + }, + })); + + // eslint-disable-next-line turbo/no-undeclared-env-vars -- NEXT_PHASE is a Next.js env variable + process.env.NEXT_PHASE = "phase-production-build"; + + const { fetchLicense } = await import("./license"); + const fetch = (await import("node-fetch")).default as Mock; + + const result = await fetchLicense(); + + expect(result).toBeNull(); + expect(fetch).not.toHaveBeenCalled(); + expect(mockCache.get).not.toHaveBeenCalled(); + }); + + test("should return null after poll timeout when fallback fetch also fails", async () => { + vi.resetModules(); + vi.doMock("@/lib/env", () => ({ + env: { + ENTERPRISE_LICENSE_KEY: "test-license-key", + ENVIRONMENT: "production", + VERCEL_URL: "some.vercel.url", + FORMBRICKS_COM_URL: "https://app.formbricks.com", + HTTPS_PROXY: undefined, + HTTP_PROXY: undefined, + }, + })); + + const { fetchLicense } = await import("./license"); + const fetch = (await import("node-fetch")).default as Mock; + + mockCache.tryLock.mockResolvedValue({ ok: true, data: false }); + mockCache.get.mockResolvedValue({ ok: true, data: null }); + mockCache.set.mockResolvedValue({ ok: true }); + + fetch.mockResolvedValueOnce({ ok: false, status: 500 } as any); + + const result = await fetchLicense(); + + expect(result).toBeNull(); + expect(mockCache.set).toHaveBeenCalledWith( + expect.stringContaining("fb:license:"), + { value: null }, + 10 * 60 * 1000 + ); + }); }); describe("getLicenseFeatures", () => { @@ -985,7 +1269,9 @@ describe("License Core Logic", () => { await getEnterpriseLicense(); await clearLicenseCache(); - expect(mockCache.del).toHaveBeenCalledWith(expect.arrayContaining([expect.stringContaining("fb:license:")])); + expect(mockCache.del).toHaveBeenCalledWith( + expect.arrayContaining([expect.stringContaining("fb:license:")]) + ); }); test("should log warning when cache.del fails", async () => { diff --git a/apps/web/modules/ee/license-check/lib/license.ts b/apps/web/modules/ee/license-check/lib/license.ts index a031923600ba..673cd398ed31 100644 --- a/apps/web/modules/ee/license-check/lib/license.ts +++ b/apps/web/modules/ee/license-check/lib/license.ts @@ -123,11 +123,16 @@ const getCacheIdentifier = () => { return hashString(env.ENTERPRISE_LICENSE_KEY); // Valid license key }; +const LICENSE_FETCH_LOCK_TTL_MS = 90 * 1000; // 90s lock so only one process fetches when cache is cold +const LICENSE_FETCH_POLL_MS = 2 * 1000; // Wait up to 2s for another process to populate cache +const LICENSE_FETCH_POLL_INTERVAL_MS = 400; + export const getCacheKeys = () => { const identifier = getCacheIdentifier(); return { FETCH_LICENSE_CACHE_KEY: createCacheKey.license.status(identifier), PREVIOUS_RESULT_CACHE_KEY: createCacheKey.license.previous_result(identifier), + FETCH_LOCK_CACHE_KEY: createCacheKey.license.fetch_lock(identifier), }; }; @@ -298,6 +303,19 @@ const handleInitialFailure = async (currentTime: Date): Promise => { + const keys = getCacheKeys(); + const result = await cache.get(keys.FETCH_LICENSE_CACHE_KEY); + if (!result.ok) return undefined; + if (result.data !== null && result.data !== undefined && "value" in result.data) { + return result.data.value; + } + return undefined; +}; + // API functions let fetchLicensePromise: Promise | null = null; @@ -426,31 +444,59 @@ export const fetchLicense = async (): Promise } fetchLicensePromise = (async () => { - // Check cache first — a single get call distinguishes "not cached" - // (data is null) from "cached null" (data is { value: null }). - const cacheKey = getCacheKeys().FETCH_LICENSE_CACHE_KEY; - const cached = await cache.get(cacheKey); - - if (cached.ok && cached.data !== null && "value" in cached.data) { - return cached.data.value; + const keys = getCacheKeys(); + const cached = await getCachedLicense(); + if (cached !== undefined) return cached; + + const lockResult = await cache.tryLock(keys.FETCH_LOCK_CACHE_KEY, "1", LICENSE_FETCH_LOCK_TTL_MS); + const acquired = lockResult.ok && lockResult.data === true; + const redisError = !lockResult.ok; + + if (acquired) { + try { + const fresh = await fetchLicenseFromServerInternal(); + const ttl = fresh ? CONFIG.CACHE.FETCH_LICENSE_TTL_MS : CONFIG.CACHE.FAILED_FETCH_TTL_MS; + + if (!fresh) { + logger.warn( + { + ttlMinutes: Math.floor(ttl / 60000), + timestamp: new Date().toISOString(), + }, + "License fetch failed, caching null result with short TTL for faster retry" + ); + } + + await cache.set(keys.FETCH_LICENSE_CACHE_KEY, { value: fresh }, ttl); + return fresh; + } finally { + // Lock expires automatically; no need to release + } } - // Cache miss -- fetch fresh - const result = await fetchLicenseFromServerInternal(); - const ttl = result ? CONFIG.CACHE.FETCH_LICENSE_TTL_MS : CONFIG.CACHE.FAILED_FETCH_TTL_MS; + // If Redis itself is down, skip the polling loop (it would just fail repeatedly) + // and go directly to fetching from the server. + if (redisError) { + logger.warn("Redis unavailable during license fetch lock; skipping poll and fetching directly"); + return await fetchLicenseFromServerInternal(); + } - if (!result) { - logger.warn( - { - ttlMinutes: Math.floor(ttl / 60000), - timestamp: new Date().toISOString(), - }, - "License fetch failed, caching null result with short TTL for faster retry" - ); + // Another process holds the lock — poll until the cache is populated or timeout + const deadline = Date.now() + LICENSE_FETCH_POLL_MS; + while (Date.now() < deadline) { + await sleep(LICENSE_FETCH_POLL_INTERVAL_MS); + const value = await getCachedLicense(); + if (value !== undefined) return value; } - await cache.set(getCacheKeys().FETCH_LICENSE_CACHE_KEY, { value: result }, ttl); - return result; + logger.warn( + { pollMs: LICENSE_FETCH_POLL_MS }, + "License cache not populated by holder within poll window; fetching in this process" + ); + const fallback = await fetchLicenseFromServerInternal(); + const fallbackTtl = fallback ? CONFIG.CACHE.FETCH_LICENSE_TTL_MS : CONFIG.CACHE.FAILED_FETCH_TTL_MS; + await cache.set(keys.FETCH_LICENSE_CACHE_KEY, { value: fallback }, fallbackTtl); + return fallback; })(); fetchLicensePromise diff --git a/packages/cache/src/cache-keys.ts b/packages/cache/src/cache-keys.ts index 91bea71bf05d..85cdf775d166 100644 --- a/packages/cache/src/cache-keys.ts +++ b/packages/cache/src/cache-keys.ts @@ -32,6 +32,7 @@ export const createCacheKey = { status: (organizationId: string): CacheKey => makeCacheKey("license", organizationId, "status"), previous_result: (organizationId: string): CacheKey => makeCacheKey("license", organizationId, "previous_result"), + fetch_lock: (organizationId: string): CacheKey => makeCacheKey("license", organizationId, "fetch_lock"), }, // Rate limiting and security