diff --git a/jest.config.js b/jest.config.js index 657a669..d861aa8 100644 --- a/jest.config.js +++ b/jest.config.js @@ -5,4 +5,7 @@ module.exports = { setupFilesAfterEnv: ['./jest.setup.ts'], collectCoverageFrom: ['src/**/*.ts', '!src/index.ts', '!src/types.ts'], coverageProvider: 'v8', + moduleNameMapper: { + "^@/(.*)$": "/src/$1", + } }; diff --git a/src/createClient.ts b/src/createClient.ts index 10e1800..aee001f 100644 --- a/src/createClient.ts +++ b/src/createClient.ts @@ -2,25 +2,26 @@ * microCMS API SDK * https://github.com/microcmsio/microcms-js-sdk */ -import { parseQuery } from './utils/parseQuery'; -import { isString } from './utils/isCheckValue'; +import { partialPromiseAll, sleep } from '@/utils/promise'; +import retry from 'async-retry'; +import { generateFetchClient } from './lib/fetch'; import { - MicroCMSClient, - MakeRequest, - GetRequest, - GetListRequest, + CreateRequest, + DeleteRequest, + GetAllContentIdsRequest, + GetAllContentRequest, GetListDetailRequest, + GetListRequest, GetObjectRequest, - WriteApiRequestResult, - CreateRequest, - MicroCMSListResponse, + GetRequest, + MakeRequest, + MicroCMSClient, MicroCMSListContent, + MicroCMSListResponse, MicroCMSObjectContent, - UpdateRequest, - DeleteRequest, - GetAllContentIdsRequest, MicroCMSQueries, - GetAllContentRequest, + UpdateRequest, + WriteApiRequestResult, } from './types'; import { API_VERSION, @@ -28,8 +29,8 @@ import { MAX_RETRY_COUNT, MIN_TIMEOUT_MS, } from './utils/constants'; -import { generateFetchClient } from './lib/fetch'; -import retry from 'async-retry'; +import { isString } from './utils/isCheckValue'; +import { parseQuery } from './utils/parseQuery'; /** * Initialize SDK Client @@ -56,12 +57,12 @@ export const createClient = ({ /** * Make request */ - const makeRequest = async ({ + const makeRequest = async >({ endpoint, contentId, queries = {}, requestInit, - }: MakeRequest) => { + }: MakeRequest): Promise => { const fetchClient = generateFetchClient(apiKey, customFetch); const queryString = parseQuery(queries); const url = `${baseUrl}/${endpoint}${contentId ? `/${contentId}` : ''}${ @@ -286,36 +287,47 @@ export const createClient = ({ endpoint, queries = {}, customRequestInit, + limit=100, + interval=1000 }: GetAllContentRequest): Promise<(T & MicroCMSListContent)[]> => { - const limit = 100; - const { totalCount } = await makeRequest({ + // info: https://document.microcms.io/manual/limitations#h9e37a059c1 + const limitPerSecond = 60; + + const { totalCount } = await makeRequest>({ endpoint, queries: { ...queries, limit: 0 }, requestInit: customRequestInit, }); + const countPerPacket = limit * limitPerSecond; + const packetCount = Math.ceil(totalCount / countPerPacket); + const requestingList = Array.from({ length: packetCount }, (_, i) => { + const baseOffset = i * countPerPacket; + return Array.from({ length: limitPerSecond }, async (_, j) => { + const offset = baseOffset + j * limit; + const { contents } = await makeRequest>({ + endpoint, + queries: { ...queries, limit, offset }, + requestInit: customRequestInit, + }); + return contents; + }) + }); - let contents: (T & MicroCMSListContent)[] = []; - let offset = 0; - - const sleep = (ms: number) => - new Promise((resolve) => setTimeout(resolve, ms)); - - while (contents.length < totalCount) { - const { contents: _contents } = (await makeRequest({ - endpoint, - queries: { ...queries, limit, offset }, - requestInit: customRequestInit, - })) as MicroCMSListResponse; - - contents = contents.concat(_contents); - - offset += limit; - if (contents.length < totalCount) { - await sleep(1000); // sleep for 1 second before the next request + const promises = await ((lastRequestDatetime=0) => requestingList.map(async (requesting, i) => { + // do not wait for the first run + if (i !== 0) { + const shouldWaitTime = lastRequestDatetime + interval - Date.now(); + if (shouldWaitTime > 0) { + await sleep(shouldWaitTime); + } } - } + const response = await Promise.all(requesting); + lastRequestDatetime = Date.now(); + return response.flat(); + }))(); + const contents = (await partialPromiseAll(promises)).flat(); return contents; }; diff --git a/src/types.ts b/src/types.ts index 17a9379..c67ae03 100644 --- a/src/types.ts +++ b/src/types.ts @@ -133,6 +133,8 @@ export interface GetAllContentRequest { endpoint: string; queries?: Omit; customRequestInit?: CustomRequestInit; + limit?: number + interval?: number } export interface WriteApiRequestResult { diff --git a/src/utils/promise.ts b/src/utils/promise.ts new file mode 100644 index 0000000..fcf70df --- /dev/null +++ b/src/utils/promise.ts @@ -0,0 +1,22 @@ +/** + * wait for milliseconds + * + * @param ms - milliseconds + * @returns {Promise} + */ +export const sleep = (ms: number) => + new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * run promises in parallel + * + * @param promises - array of promises + * @returns {T[]} + */ +export const partialPromiseAll = async (promises: Promise[]) => { + const results = promises.reduce(async (acc: Promise, promise: Promise) => { + const result = await promise; + return [...(await acc), result]; + }, Promise.resolve([])); + return results; +}; \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index bf43d30..fe385b3 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,4 +1,5 @@ { + "baseUrl": "./", "compilerOptions": { "target": "es5", "module": "es6", @@ -11,7 +12,10 @@ "declaration": true, "strict": true, "esModuleInterop": true, - "useUnknownInCatchVariables": false + "useUnknownInCatchVariables": false, + "paths": { + "@/*": ["src/*"] + } }, "include": ["src/**/*", "test/**/*"], "exclude": ["node_modules"]