getAllContents関数のリファクタ

jest.config.js

@@ -5,4 +5,7 @@ module.exports = {
   setupFilesAfterEnv: ['./jest.setup.ts'],
   collectCoverageFrom: ['src/**/*.ts', '!src/index.ts', '!src/types.ts'],
   coverageProvider: 'v8',
+  moduleNameMapper: {
+    "^@/(.*)$": "<rootDir>/src/$1",
+  }
 };

src/createClient.ts

@@ -2,34 +2,35 @@
  * microCMS API SDK
  * https://github.com/microcmsio/microcms-js-sdk
  */
-import { parseQuery } from './utils/parseQuery';
-import { isString } from './utils/isCheckValue';
+import { partialPromiseAll, sleep } from '@/utils/promise';
+import retry from 'async-retry';
+import { generateFetchClient } from './lib/fetch';
 import {
-  MicroCMSClient,
-  MakeRequest,
-  GetRequest,
-  GetListRequest,
+  CreateRequest,
+  DeleteRequest,
+  GetAllContentIdsRequest,
+  GetAllContentRequest,
   GetListDetailRequest,
+  GetListRequest,
   GetObjectRequest,
-  WriteApiRequestResult,
-  CreateRequest,
-  MicroCMSListResponse,
+  GetRequest,
+  MakeRequest,
+  MicroCMSClient,
   MicroCMSListContent,
+  MicroCMSListResponse,
   MicroCMSObjectContent,
-  UpdateRequest,
-  DeleteRequest,
-  GetAllContentIdsRequest,
   MicroCMSQueries,
-  GetAllContentRequest,
+  UpdateRequest,
+  WriteApiRequestResult,
 } from './types';
 import {
   API_VERSION,
   BASE_DOMAIN,
   MAX_RETRY_COUNT,
   MIN_TIMEOUT_MS,
 } from './utils/constants';
-import { generateFetchClient } from './lib/fetch';
-import retry from 'async-retry';
+import { isString } from './utils/isCheckValue';
+import { parseQuery } from './utils/parseQuery';
 
 /**
  * Initialize SDK Client
@@ -56,12 +57,12 @@ export const createClient = ({
   /**
    * Make request
    */
-  const makeRequest = async ({
+  const makeRequest = async <T=MicroCMSListResponse<any>>({
     endpoint,
     contentId,
     queries = {},
     requestInit,
-  }: MakeRequest) => {
+  }: MakeRequest): Promise<T> => {
     const fetchClient = generateFetchClient(apiKey, customFetch);
     const queryString = parseQuery(queries);
     const url = `${baseUrl}/${endpoint}${contentId ? `/${contentId}` : ''}${
@@ -286,36 +287,47 @@ export const createClient = ({
     endpoint,
     queries = {},
     customRequestInit,
+    limit=100,
+    interval=1000
   }: GetAllContentRequest): Promise<(T & MicroCMSListContent)[]> => {
-    const limit = 100;
 
-    const { totalCount } = await makeRequest({
+    // info: https://document.microcms.io/manual/limitations#h9e37a059c1
+    const limitPerSecond = 60;
+
+    const { totalCount } = await makeRequest<MicroCMSListResponse<T>>({
       endpoint,
       queries: { ...queries, limit: 0 },
       requestInit: customRequestInit,
     });
+    const countPerPacket = limit * limitPerSecond;
+    const packetCount = Math.ceil(totalCount / countPerPacket);
+    const requestingList = Array.from({ length: packetCount }, (_, i) => {
+      const baseOffset = i * countPerPacket;
+      return Array.from({ length: limitPerSecond }, async (_, j) => {
+        const offset = baseOffset + j * limit;
+        const { contents } = await makeRequest<MicroCMSListResponse<T>>({
+          endpoint,
+          queries: { ...queries, limit, offset },
+          requestInit: customRequestInit,
+        });
+        return contents;
+      })
+    });
 
-    let contents: (T & MicroCMSListContent)[] = [];
-    let offset = 0;
-
-    const sleep = (ms: number) =>
-      new Promise((resolve) => setTimeout(resolve, ms));
-
-    while (contents.length < totalCount) {
-      const { contents: _contents } = (await makeRequest({
-        endpoint,
-        queries: { ...queries, limit, offset },
-        requestInit: customRequestInit,
-      })) as MicroCMSListResponse<T>;
-
-      contents = contents.concat(_contents);
-
-      offset += limit;
-      if (contents.length < totalCount) {
-        await sleep(1000); // sleep for 1 second before the next request
+    const promises = await ((lastRequestDatetime=0) => requestingList.map(async (requesting, i) => {
+      // do not wait for the first run
+      if (i !== 0) {
+        const shouldWaitTime = lastRequestDatetime + interval - Date.now();
+        if (shouldWaitTime > 0) {
+          await sleep(shouldWaitTime);
+        }
       }
-    }
+      const response = await Promise.all(requesting);
+      lastRequestDatetime = Date.now();
+      return response.flat();
+    }))();
 
+    const contents = (await partialPromiseAll(promises)).flat();
     return contents;
   };
 

src/types.ts

@@ -133,6 +133,8 @@ export interface GetAllContentRequest {
   endpoint: string;
   queries?: Omit<MicroCMSQueries, 'limit' | 'offset' | 'ids'>;
   customRequestInit?: CustomRequestInit;
+  limit?: number
+  interval?: number
 }
 
 export interface WriteApiRequestResult {

src/utils/promise.ts

@@ -0,0 +1,22 @@
+/**
+ * wait for milliseconds
+ * 
+ * @param ms - milliseconds
+ * @returns {Promise<void>}
+ */
+export const sleep = (ms: number) =>
+  new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * run promises in parallel
+ * 
+ * @param promises - array of promises
+ * @returns {T[]}
+ */
+export const partialPromiseAll = async <T>(promises: Promise<T>[]) => {
+  const results = promises.reduce(async (acc: Promise<T[]>, promise: Promise<T>) => {
+    const result = await promise;
+    return [...(await acc), result];
+  }, Promise.resolve([]));
+  return results;
+};

tsconfig.json

@@ -1,4 +1,5 @@
 {
+  "baseUrl": "./",  
   "compilerOptions": {
     "target": "es5",
     "module": "es6",
@@ -11,7 +12,10 @@
     "declaration": true,
     "strict": true,
     "esModuleInterop": true,
-    "useUnknownInCatchVariables": false
+    "useUnknownInCatchVariables": false,
+    "paths": {
+      "@/*": ["src/*"]
+    }
   },
   "include": ["src/**/*", "test/**/*"],
   "exclude": ["node_modules"]