Merge pull request #179 from contentstack/fix/DX-3851

feat: add url length estimation and compact format support for parame…

Author: nadeem-cs

CHANGELOG.md

@@ -1,5 +1,9 @@
 ## Change log
 
+### Version: 1.3.9
+#### Date: Jan-26-2026
+ - Fix: add url length estimation and compact format support for parameter serialization
+
 ### Version: 1.3.8
 #### Date: Jan-12-2026
  - Fix: Add .js extensions to relative imports in ESM build for proper module resolution

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@contentstack/core",
-  "version": "1.3.8",
+  "version": "1.3.9",
   "type": "commonjs",
   "main": "./dist/cjs/src/index.js",
   "types": "./dist/cjs/src/index.d.ts",

package.json

@@ -25,6 +25,9 @@ export const ERROR_MESSAGES = {
   // Request Handler Messages
   REQUEST: {
     HOST_REQUIRED_FOR_LIVE_PREVIEW: 'Host is required for live preview. Please provide a valid host parameter in the live_preview configuration.',
+    URL_TOO_LONG: (urlLength: number, maxLength: number) => 
+      `Request URL length (${urlLength} characters) exceeds the maximum allowed length (${maxLength} characters). ` +
+      'Please reduce the number of includeReference parameters or split your request into multiple smaller requests.',
   },
 
   // Retry Policy Messages

src/lib/error-messages.ts

@@ -1,9 +1,22 @@
 import * as Qs from 'qs';
+import { ParamsSerializerOptions } from 'axios';
+
+interface ExtendedParamsSerializerOptions extends ParamsSerializerOptions {
+  useCompactFormat?: boolean;
+}
+
+export function serialize(
+  params: Record<string, any>,
+  options?: ParamsSerializerOptions | ExtendedParamsSerializerOptions | boolean
+): string {
+  // Support both axios signature (options object) and legacy signature (boolean)
+  const useCompactFormat =
+    typeof options === 'boolean' ? options : (options as ExtendedParamsSerializerOptions)?.useCompactFormat ?? false;
 
-export function serialize(params: Record<string, any>) {
   const query = params.query;
   delete params.query;
-  let qs = Qs.stringify(params, { arrayFormat: 'brackets' });
+  const arrayFormat = useCompactFormat ? 'comma' : 'brackets';
+  let qs = Qs.stringify(params, { arrayFormat });
   if (query) {
     qs = qs + `&query=${encodeURI(JSON.stringify(query))}`;
   }

src/lib/param-serializer.ts

@@ -7,10 +7,33 @@ import { ERROR_MESSAGES } from './error-messages';
  * Handles array parameters properly with & separators
  * React Native compatible implementation without URLSearchParams.set()
  */
-function serializeParams(params: any): string {
+function serializeParams(params: any, useCompactFormat = false): string {
   if (!params) return '';
 
-  return serialize(params);
+  return serialize(params, { useCompactFormat } as any);
+}
+
+/**
+ * Estimates the URL length that would be generated from the given parameters
+ * @param baseURL - The base URL
+ * @param url - The endpoint URL
+ * @param params - The query parameters
+ * @param useCompactFormat - Whether to use compact format for estimation
+ * @returns Estimated URL length
+ */
+function estimateUrlLength(baseURL: string | undefined, url: string, params: any, useCompactFormat = false): number {
+  if (!params) {
+    const base = baseURL || '';
+
+    return (url.startsWith('http://') || url.startsWith('https://') ? url : `${base}${url}`).length;
+  }
+
+  const queryString = serializeParams(params, useCompactFormat);
+  const base = baseURL || '';
+  const fullUrl =
+    url.startsWith('http://') || url.startsWith('https://') ? `${url}?${queryString}` : `${base}${url}?${queryString}`;
+
+  return fullUrl.length;
 }
 
 /**
@@ -59,6 +82,13 @@ function handleRequestError(err: any): Error {
 
 export async function getData(instance: AxiosInstance, url: string, data?: any) {
   try {
+    let isLivePreview = false;
+    let livePreviewUrl = url;
+
+    if (!data) {
+      data = {};
+    }
+
     if (instance.stackConfig && instance.stackConfig.live_preview) {
       const livePreviewParams = instance.stackConfig.live_preview;
 
@@ -76,7 +106,9 @@ export async function getData(instance: AxiosInstance, url: string, data?: any)
           if (!livePreviewParams.host) {
             throw new Error(ERROR_MESSAGES.REQUEST.HOST_REQUIRED_FOR_LIVE_PREVIEW);
           }
-          url = (livePreviewParams.host.startsWith('https://') ? '' : 'https://') + livePreviewParams.host + url;
+          livePreviewUrl =
+            (livePreviewParams.host.startsWith('https://') ? '' : 'https://') + livePreviewParams.host + url;
+          isLivePreview = true;
         }
       }
     }
@@ -85,10 +117,41 @@ export async function getData(instance: AxiosInstance, url: string, data?: any)
       ...data,
       maxContentLength: Infinity,
       maxBodyLength: Infinity,
-    };
-    const queryString = serializeParams(requestConfig.params);
-    const actualFullUrl = buildFullUrl(instance.defaults.baseURL, url, queryString);
-    const response = await makeRequest(instance, url, requestConfig, actualFullUrl);
+    } as any;
+
+    // Determine URL length thresholds
+    // Use lower threshold for Live Preview (1500) vs regular requests (2000)
+    const maxUrlLength = isLivePreview ? 1500 : 2000;
+    const baseURLForEstimation = isLivePreview ? undefined : instance.defaults.baseURL;
+    const urlForEstimation = isLivePreview ? livePreviewUrl : url;
+
+    // Estimate URL length with standard format
+    const estimatedLength = estimateUrlLength(baseURLForEstimation, urlForEstimation, requestConfig.params, false);
+    let useCompactFormat = false;
+
+    // If URL would exceed threshold, try compact format
+    if (estimatedLength > maxUrlLength) {
+      const compactEstimatedLength = estimateUrlLength(
+        baseURLForEstimation,
+        urlForEstimation,
+        requestConfig.params,
+        true
+      );
+      if (compactEstimatedLength <= maxUrlLength) {
+        useCompactFormat = true;
+      } else {
+        // Even with compact format, URL is too long
+        throw new Error(ERROR_MESSAGES.REQUEST.URL_TOO_LONG(compactEstimatedLength, maxUrlLength));
+      }
+    }
+
+    const queryString = serializeParams(requestConfig.params, useCompactFormat);
+    const actualFullUrl = buildFullUrl(
+      isLivePreview ? undefined : instance.defaults.baseURL,
+      isLivePreview ? livePreviewUrl : url,
+      queryString
+    );
+    const response = await makeRequest(instance, isLivePreview ? livePreviewUrl : url, requestConfig, actualFullUrl);
 
     if (response && response.data) {
       return response.data;

src/lib/request.ts

@@ -32,4 +32,23 @@ describe('serialize', () => {
     );
     done();
   });
+
+  it('should return comma-separated format when useCompactFormat is true', (done) => {
+    const param = serialize({ 'include[]': ['ref1', 'ref2', 'ref3'] }, true);
+    expect(param).toEqual('include=ref1%2Cref2%2Cref3');
+    done();
+  });
+
+  it('should return brackets format when useCompactFormat is false', (done) => {
+    const param = serialize({ 'include[]': ['ref1', 'ref2', 'ref3'] }, false);
+    expect(param).toEqual('include%5B%5D=ref1&include%5B%5D=ref2&include%5B%5D=ref3');
+    done();
+  });
+
+  it('should handle query param with compact format', (done) => {
+    const param = serialize({ 'include[]': ['ref1', 'ref2'], query: { title: 'test' } }, true);
+    expect(param).toContain('include=ref1%2Cref2');
+    expect(param).toContain('query=');
+    done();
+  });
 });

test/param-serializer.spec.ts

@@ -529,4 +529,98 @@ describe('Request tests', () => {
       requestSpy.mockRestore();
     });
   });
+
+  describe('URL length optimization for includeReference parameters', () => {
+    it('should use compact format when URL with many include[] parameters exceeds threshold', async () => {
+      const client = httpClient({ defaultHostname: 'example.com' });
+      const mock = new MockAdapter(client as any);
+      const url = '/content_types/blog/entries/entry123';
+      const mockResponse = { entry: { uid: 'entry123', title: 'Test' } };
+
+      // Create many include[] parameters that would make URL long
+      const manyIncludes = Array.from({ length: 100 }, (_, i) => `ref_field_${i}`);
+      const requestData = { params: { 'include[]': manyIncludes } };
+
+      mock.onGet(url).reply(200, mockResponse);
+
+      const result = await getData(client, url, requestData);
+      expect(result).toEqual(mockResponse);
+      
+      // Verify the request was made (URL optimization allowed it to succeed)
+      expect(mock.history.get.length).toBe(1);
+      const requestUrl = mock.history.get[0].url || '';
+      // With compact format, the URL should be shorter and contain comma-separated values
+      // We verify success means the optimization worked
+      expect(requestUrl.length).toBeLessThan(3000);
+    });
+
+    it('should use compact format for Live Preview requests with lower threshold', async () => {
+      const client = httpClient({ defaultHostname: 'example.com' });
+      const mock = new MockAdapter(client as any);
+      const url = '/content_types/blog/entries/entry123';
+      const mockResponse = { entry: { uid: 'entry123', title: 'Test' } };
+
+      client.stackConfig = {
+        live_preview: {
+          enable: true,
+          preview_token: 'someToken',
+          live_preview: 'someHash',
+          host: 'rest-preview.contentstack.com',
+        },
+      };
+
+      // Create include[] parameters that would exceed 1500 chars for Live Preview
+      // but might be okay for regular requests (2000 chars)
+      const manyIncludes = Array.from({ length: 80 }, (_, i) => `ref_field_${i}_with_long_name`);
+      const requestData = { params: { 'include[]': manyIncludes } };
+
+      const livePreviewUrl = 'https://rest-preview.contentstack.com' + url;
+      mock.onGet(livePreviewUrl).reply(200, mockResponse);
+
+      const result = await getData(client, url, requestData);
+      expect(result).toEqual(mockResponse);
+      
+      // Verify the request was made to Live Preview host
+      expect(mock.history.get.length).toBe(1);
+      expect(mock.history.get[0].url).toContain('rest-preview.contentstack.com');
+    });
+
+    it('should throw error when URL is too long even with compact format', async () => {
+      const client = httpClient({ defaultHostname: 'example.com' });
+      const url = '/content_types/blog/entries/entry123';
+
+      client.stackConfig = {
+        live_preview: {
+          enable: true,
+          preview_token: 'someToken',
+          live_preview: 'someHash',
+          host: 'rest-preview.contentstack.com',
+        },
+      };
+
+      // Create an extremely large number of includes that would exceed even compact format
+      const manyIncludes = Array.from({ length: 500 }, (_, i) => `very_long_reference_field_name_${i}_with_many_characters`);
+      const requestData = { params: { 'include[]': manyIncludes } };
+
+      await expect(getData(client, url, requestData)).rejects.toThrow(/exceeds the maximum allowed length/);
+    });
+
+    it('should use standard format when URL length is within threshold', async () => {
+      const client = httpClient({ defaultHostname: 'example.com' });
+      const mock = new MockAdapter(client as any);
+      const url = '/content_types/blog/entries/entry123';
+      const mockResponse = { entry: { uid: 'entry123', title: 'Test' } };
+
+      // Create a small number of includes that won't exceed threshold
+      const requestData = { params: { 'include[]': ['ref1', 'ref2', 'ref3'] } };
+
+      mock.onGet(url).reply(200, mockResponse);
+
+      const result = await getData(client, url, requestData);
+      expect(result).toEqual(mockResponse);
+      
+      // Verify the request was made successfully
+      expect(mock.history.get.length).toBe(1);
+    });
+  });
 });