diff --git a/README.md b/README.md
index e035cc8..2a265de 100644
--- a/README.md
+++ b/README.md
@@ -18,7 +18,7 @@ This library compares two arrays or objects and returns a full diff of their dif
Most existing solutions return a confusing diff format that often requires extra parsing. They are also limited to object comparison.
-**Superdiff** provides a complete and readable diff for both arrays **and** objects. Plus, it's battle-tested, has zero dependencies, and is super fast.
+**Superdiff** provides a complete and readable diff for both arrays **and** objects. Plus, it supports stream and file inputs for handling large datasets efficiently, is battle-tested, has zero dependencies, and is super fast.
Import. Enjoy. 👍
@@ -40,7 +40,7 @@ I am grateful to the generous donors of **Superdiff**!
## FEATURES
-**Superdiff** exports 5 functions:
+**Superdiff** exports 6 functions:
```ts
// Returns a complete diff of two objects
@@ -52,6 +52,9 @@ getListDiff(prevList, nextList)
// Streams the diff of two object lists, ideal for large lists and maximum performance
streamListDiff(prevList, nextList, referenceProperty)
+// Similar to streamListDiff, but for browser use
+streamListDiffClient(prevList, nextList, referenceProperty)
+
// Checks whether two values are equal
isEqual(dataA, dataB)
@@ -306,7 +309,10 @@ getListDiff(
### streamListDiff()
```js
+// If you are in a server environment
import { streamListDiff } from "@donedeal0/superdiff";
+// If you are in a browser environment
+import { streamListDiffClient } from "@donedeal0/superdiff";
```
Streams the diff of two object lists, ideal for large lists and maximum performance.
@@ -315,14 +321,34 @@ Streams the diff of two object lists, ideal for large lists and maximum performa
**Input**
+#### streamListDiff (server)
+
+> In a server environment, `Readable` refers to Node.js streams, and `FilePath` refers to the path of a file (e.g., `./list.json`). Examples are provided in the #usage section below.
+
```ts
- prevList: Record<string, unknown>[],
- nextList: Record<string, unknown>[],
+// streamListDiff
+ prevList: Readable | FilePath | Record<string, unknown>[],
+ nextList: Readable | FilePath | Record<string, unknown>[],
referenceProperty: keyof Record<string, unknown>,
options: {
showOnly?: ("added" | "deleted" | "moved" | "updated" | "equal")[], // [] by default
- chunksSize?: number, // // 0 by default
- considerMoveAsUpdate? boolean; // false by default
+ chunksSize?: number, // 0 by default
+ considerMoveAsUpdate?: boolean; // false by default
+}
+```
+
+#### streamListDiffClient (browser)
+
+> In a browser environment, `ReadableStream` refers to the browser's streaming API, and `File` refers to an uploaded or local file. Examples are provided in the #usage section below.
+
+```ts
+ prevList: ReadableStream<Record<string, unknown>> | File | Record<string, unknown>[],
+ nextList: ReadableStream<Record<string, unknown>> | File | Record<string, unknown>[],
+ referenceProperty: keyof Record<string, unknown>,
+ options: {
+ showOnly?: ("added" | "deleted" | "moved" | "updated" | "equal")[], // [] by default
+ chunksSize?: number, // 0 by default
+ considerMoveAsUpdate?: boolean; // false by default
}
```
@@ -370,6 +396,40 @@ type StreamListDiff<T extends Record<string, unknown>> = {
**Input**
+You can send streams, file paths, or arrays as input:
+
+> If you are in a server environment
+
+```ts
+ // for a simple array
+ const stream = [{ id: 1, name: "hello" }]
+ // for a large array
+ const stream = Readable.from(list, { objectMode: true });
+ // for a local file
+ const stream = path.resolve(__dirname, "./list.json");
+
+```
+
+> If you are in a browser environment
+
+```ts
+ // for a simple array
+ const stream = [{ id: 1, name: "hello" }]
+ // for a large array
+ const stream = new ReadableStream({
+ start(controller) {
+ list.forEach((value) => controller.enqueue(value));
+ controller.close();
+ },
+ });
+ // for a local file
+ const stream = new File([JSON.stringify(file)], "file.json", { type: "application/json" });
+ // for a file input
+ const stream = e.target.files[0];
+
+```
+> Example
+
```diff
const diff = streamListDiff(
[
@@ -431,9 +491,78 @@ diff.on("data", (chunk) => {
]
});
-diff.on("finish", () => console.log("The full diff is available"))
+diff.on("finish", () => console.log("Your data has been processed. The full diff is available."))
diff.on("error", (err) => console.log(err))
```
+
+**Using `fetch`**
+
+A common use case would be to do a live diff against a stream, in order to avoid loading the entire dataset into memory. Here are two examples, for browser and server use:
+
+Browser:
+```ts
+import { streamListDiffClient } from "@donedeal0/superdiff";
+
+async function streamDiffFromAPI() {
+ try {
+ const response = await fetch("https://example.com/api/streaming-data");
+ const reader = response.body.getReader();
+
+ const stream = new ReadableStream({
+ async start(controller) {
+ let result;
+ while (!(result = await reader.read()).done) {
+ controller.enqueue(result.value); // Push the next chunk into the stream
+ }
+ controller.close(); // Close the stream when done
+ },
+ });
+
+ const prevStream = [{ id: 1, name: "Joe" }, { id: 2, name: "Jane" }] // Some previous list or stream
+
+ const diff = streamListDiffClient(prevStream, stream, 'id', { chunksSize: 5 });
+ diff.on("data", (diffChunk) => console.log(diffChunk));
+ diff.on("finish", () => console.log("Stream diff complete"));
+ } catch (err) {
+ console.error(err);
+ }
+}
+```
+
+Server:
+
+```ts
+import fetch from "node-fetch";
+import { Readable } from "stream";
+import { streamListDiff } from "@donedeal0/superdiff";
+
+async function streamDiffFromAPI() {
+ try {
+ const response = await fetch("https://example.com/api/streaming-data");
+ const reader = response.body.getReader();
+
+ const stream = new Readable({
+ async read() {
+ let result;
+ while (!(result = await reader.read()).done) {
+ this.push(result.value); // Push the next chunk into the stream
+ }
+ this.push(null); // Close the stream when done
+ },
+ });
+
+ const prevList = [{ id: 1, name: "Joe" }, { id: 2, name: "Jane" }]; // Some previous list or stream
+ const prevListStream = Readable.from(prevList, { objectMode: true })
+ const diff = streamListDiff(prevListStream, stream, 'id', { chunksSize: 5 });
+
+ diff.on("data", (diffChunk) => console.log(diffChunk));
+ diff.on("finish", () => console.log("Stream diff complete"));
+ } catch (err) {
+ console.error(err);
+ }
+}
+```
+
<hr/>
### isEqual()
diff --git a/package-lock.json b/package-lock.json
index e2eae4f..ef1fda1 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@donedeal0/superdiff",
- "version": "2.0.0",
+ "version": "2.1.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@donedeal0/superdiff",
- "version": "2.0.0",
+ "version": "2.1.0",
"license": "ISC",
"devDependencies": {
"@eslint/js": "^9.11.1",
@@ -17,14 +17,21 @@
"@swc/core": "^1.7.26",
"@swc/jest": "^0.2.36",
"@types/jest": "^29.5.13",
+ "blob-polyfill": "^9.0.20240710",
"eslint": "^9.11.1",
"husky": "^9.1.6",
"jest": "^29.7.0",
+ "jest-environment-jsdom": "^29.7.0",
"prettier": "^3.3.3",
"swc-loader": "^0.2.6",
"tsup": "^8.3.0",
"typescript": "^5.6.2",
- "typescript-eslint": "^8.7.0"
+ "typescript-eslint": "^8.7.0",
+ "web-streams-polyfill": "^4.0.0"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/DoneDeal0"
}
},
"node_modules/@ampproject/remapping": {
@@ -3073,6 +3080,15 @@
"@swc/counter": "^0.1.3"
}
},
+ "node_modules/@tootallnate/once": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz",
+ "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==",
+ "dev": true,
+ "engines": {
+ "node": ">= 10"
+ }
+ },
"node_modules/@types/babel__core": {
"version": "7.20.5",
"resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz",
@@ -3163,6 +3179,29 @@
"pretty-format": "^29.0.0"
}
},
+ "node_modules/@types/jsdom": {
+ "version": "20.0.1",
+ "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-20.0.1.tgz",
+ "integrity": "sha512-d0r18sZPmMQr1eG35u12FZfhIXNrnsPU/g5wvRKCUf/tOGilKKwYMYGqh33BNR6ba+2gkHw1EUiHoN3mn7E5IQ==",
+ "dev": true,
+ "dependencies": {
+ "@types/node": "*",
+ "@types/tough-cookie": "*",
+ "parse5": "^7.0.0"
+ }
+ },
+ "node_modules/@types/jsdom/node_modules/parse5": {
+ "version": "7.2.0",
+ "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.2.0.tgz",
+ "integrity": "sha512-ZkDsAOcxsUMZ4Lz5fVciOehNcJ+Gb8gTzcA4yl3wnc273BAybYWrQ+Ks/OjCjSEpjvQkDSeZbybK9qj2VHHdGA==",
+ "dev": true,
+ "dependencies": {
+ "entities": "^4.5.0"
+ },
+ "funding": {
+ "url": "https://github.com/inikulin/parse5?sponsor=1"
+ }
+ },
"node_modules/@types/json-schema": {
"version": "7.0.15",
"resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz",
@@ -3194,6 +3233,12 @@
"integrity": "sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==",
"dev": true
},
+ "node_modules/@types/tough-cookie": {
+ "version": "4.0.5",
+ "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz",
+ "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==",
+ "dev": true
+ },
"node_modules/@types/yargs": {
"version": "17.0.17",
"resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.17.tgz",
@@ -3614,6 +3659,13 @@
"dev": true,
"peer": true
},
+ "node_modules/abab": {
+ "version": "2.0.6",
+ "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz",
+ "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==",
+ "deprecated": "Use your platform's native atob() and btoa() methods instead",
+ "dev": true
+ },
"node_modules/acorn": {
"version": "8.12.1",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz",
@@ -3626,6 +3678,16 @@
"node": ">=0.4.0"
}
},
+ "node_modules/acorn-globals": {
+ "version": "7.0.1",
+ "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-7.0.1.tgz",
+ "integrity": "sha512-umOSDSDrfHbTNPuNpC2NSnnA3LUrqpevPb4T9jRx4MagXNS0rs+gwiTcAvqCRmsD6utzsrzNt+ebm00SNWiC3Q==",
+ "dev": true,
+ "dependencies": {
+ "acorn": "^8.1.0",
+ "acorn-walk": "^8.0.2"
+ }
+ },
"node_modules/acorn-import-attributes": {
"version": "1.9.5",
"resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz",
@@ -3645,6 +3707,18 @@
"acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"
}
},
+ "node_modules/acorn-walk": {
+ "version": "8.3.4",
+ "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz",
+ "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==",
+ "dev": true,
+ "dependencies": {
+ "acorn": "^8.11.0"
+ },
+ "engines": {
+ "node": ">=0.4.0"
+ }
+ },
"node_modules/agent-base": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz",
@@ -3774,6 +3848,12 @@
"dev": true,
"peer": true
},
+ "node_modules/asynckit": {
+ "version": "0.4.0",
+ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
+ "dev": true
+ },
"node_modules/babel-jest": {
"version": "29.7.0",
"resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz",
@@ -3959,6 +4039,12 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
+ "node_modules/blob-polyfill": {
+ "version": "9.0.20240710",
+ "resolved": "https://registry.npmjs.org/blob-polyfill/-/blob-polyfill-9.0.20240710.tgz",
+ "integrity": "sha512-DPUO/EjNANCgSVg0geTy1vmUpu5hhp9tV2F7xUSTUd1jwe4XpwupGB+lt5PhVUqpqAk+zK1etqp6Pl/HVf71Ug==",
+ "dev": true
+ },
"node_modules/bottleneck": {
"version": "2.19.5",
"resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz",
@@ -4389,6 +4475,18 @@
"integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==",
"dev": true
},
+ "node_modules/combined-stream": {
+ "version": "1.0.8",
+ "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+ "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+ "dev": true,
+ "dependencies": {
+ "delayed-stream": "~1.0.0"
+ },
+ "engines": {
+ "node": ">= 0.8"
+ }
+ },
"node_modules/commander": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz",
@@ -4711,6 +4809,78 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
+ "node_modules/cssom": {
+ "version": "0.5.0",
+ "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz",
+ "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==",
+ "dev": true
+ },
+ "node_modules/cssstyle": {
+ "version": "2.3.0",
+ "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz",
+ "integrity": "sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==",
+ "dev": true,
+ "dependencies": {
+ "cssom": "~0.3.6"
+ },
+ "engines": {
+ "node": ">=8"
+ }
+ },
+ "node_modules/cssstyle/node_modules/cssom": {
+ "version": "0.3.8",
+ "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz",
+ "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==",
+ "dev": true
+ },
+ "node_modules/data-urls": {
+ "version": "3.0.2",
+ "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz",
+ "integrity": "sha512-Jy/tj3ldjZJo63sVAvg6LHt2mHvl4V6AgRAmNDtLdm7faqtsx+aJG42rsyCo9JCoRVKwPFzKlIPx3DIibwSIaQ==",
+ "dev": true,
+ "dependencies": {
+ "abab": "^2.0.6",
+ "whatwg-mimetype": "^3.0.0",
+ "whatwg-url": "^11.0.0"
+ },
+ "engines": {
+ "node": ">=12"
+ }
+ },
+ "node_modules/data-urls/node_modules/tr46": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz",
+ "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==",
+ "dev": true,
+ "dependencies": {
+ "punycode": "^2.1.1"
+ },
+ "engines": {
+ "node": ">=12"
+ }
+ },
+ "node_modules/data-urls/node_modules/webidl-conversions": {
+ "version": "7.0.0",
+ "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
+ "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",
+ "dev": true,
+ "engines": {
+ "node": ">=12"
+ }
+ },
+ "node_modules/data-urls/node_modules/whatwg-url": {
+ "version": "11.0.0",
+ "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz",
+ "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==",
+ "dev": true,
+ "dependencies": {
+ "tr46": "^3.0.0",
+ "webidl-conversions": "^7.0.0"
+ },
+ "engines": {
+ "node": ">=12"
+ }
+ },
"node_modules/debug": {
"version": "4.3.7",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz",
@@ -4728,6 +4898,12 @@
}
}
},
+ "node_modules/decimal.js": {
+ "version": "10.4.3",
+ "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz",
+ "integrity": "sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==",
+ "dev": true
+ },
"node_modules/dedent": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/dedent/-/dedent-1.5.1.tgz",
@@ -4766,6 +4942,15 @@
"node": ">=0.10.0"
}
},
+ "node_modules/delayed-stream": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+ "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+ "dev": true,
+ "engines": {
+ "node": ">=0.4.0"
+ }
+ },
"node_modules/detect-newline": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz",
@@ -4796,6 +4981,28 @@
"node": ">=8"
}
},
+ "node_modules/domexception": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/domexception/-/domexception-4.0.0.tgz",
+ "integrity": "sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==",
+ "deprecated": "Use your platform's native DOMException instead",
+ "dev": true,
+ "dependencies": {
+ "webidl-conversions": "^7.0.0"
+ },
+ "engines": {
+ "node": ">=12"
+ }
+ },
+ "node_modules/domexception/node_modules/webidl-conversions": {
+ "version": "7.0.0",
+ "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
+ "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",
+ "dev": true,
+ "engines": {
+ "node": ">=12"
+ }
+ },
"node_modules/dot-prop": {
"version": "5.3.0",
"resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.3.0.tgz",
@@ -4870,6 +5077,18 @@
"node": ">=10.13.0"
}
},
+ "node_modules/entities": {
+ "version": "4.5.0",
+ "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
+ "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
+ "dev": true,
+ "engines": {
+ "node": ">=0.12"
+ },
+ "funding": {
+ "url": "https://github.com/fb55/entities?sponsor=1"
+ }
+ },
"node_modules/env-ci": {
"version": "11.1.0",
"resolved": "https://registry.npmjs.org/env-ci/-/env-ci-11.1.0.tgz",
@@ -5124,6 +5343,27 @@
"node": ">=0.8.0"
}
},
+ "node_modules/escodegen": {
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz",
+ "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==",
+ "dev": true,
+ "dependencies": {
+ "esprima": "^4.0.1",
+ "estraverse": "^5.2.0",
+ "esutils": "^2.0.2"
+ },
+ "bin": {
+ "escodegen": "bin/escodegen.js",
+ "esgenerate": "bin/esgenerate.js"
+ },
+ "engines": {
+ "node": ">=6.0"
+ },
+ "optionalDependencies": {
+ "source-map": "~0.6.1"
+ }
+ },
"node_modules/eslint": {
"version": "9.11.1",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-9.11.1.tgz",
@@ -5665,6 +5905,20 @@
"url": "https://github.com/sponsors/isaacs"
}
},
+ "node_modules/form-data": {
+ "version": "4.0.1",
+ "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz",
+ "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==",
+ "dev": true,
+ "dependencies": {
+ "asynckit": "^0.4.0",
+ "combined-stream": "^1.0.8",
+ "mime-types": "^2.1.12"
+ },
+ "engines": {
+ "node": ">= 6"
+ }
+ },
"node_modules/from2": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz",
@@ -5974,6 +6228,18 @@
"integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==",
"dev": true
},
+ "node_modules/html-encoding-sniffer": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz",
+ "integrity": "sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==",
+ "dev": true,
+ "dependencies": {
+ "whatwg-encoding": "^2.0.0"
+ },
+ "engines": {
+ "node": ">=12"
+ }
+ },
"node_modules/html-escaper": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz",
@@ -6030,6 +6296,18 @@
"url": "https://github.com/sponsors/typicode"
}
},
+ "node_modules/iconv-lite": {
+ "version": "0.6.3",
+ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+ "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+ "dev": true,
+ "dependencies": {
+ "safer-buffer": ">= 2.1.2 < 3.0.0"
+ },
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
"node_modules/ignore": {
"version": "5.3.2",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz",
@@ -6286,6 +6564,12 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
+ "node_modules/is-potential-custom-element-name": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz",
+ "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==",
+ "dev": true
+ },
"node_modules/is-stream": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
@@ -6993,6 +7277,33 @@
"node": ">=8"
}
},
+ "node_modules/jest-environment-jsdom": {
+ "version": "29.7.0",
+ "resolved": "https://registry.npmjs.org/jest-environment-jsdom/-/jest-environment-jsdom-29.7.0.tgz",
+ "integrity": "sha512-k9iQbsf9OyOfdzWH8HDmrRT0gSIcX+FLNW7IQq94tFX0gynPwqDTW0Ho6iMVNjGz/nb+l/vW3dWM2bbLLpkbXA==",
+ "dev": true,
+ "dependencies": {
+ "@jest/environment": "^29.7.0",
+ "@jest/fake-timers": "^29.7.0",
+ "@jest/types": "^29.6.3",
+ "@types/jsdom": "^20.0.0",
+ "@types/node": "*",
+ "jest-mock": "^29.7.0",
+ "jest-util": "^29.7.0",
+ "jsdom": "^20.0.0"
+ },
+ "engines": {
+ "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
+ },
+ "peerDependencies": {
+ "canvas": "^2.5.0"
+ },
+ "peerDependenciesMeta": {
+ "canvas": {
+ "optional": true
+ }
+ }
+ },
"node_modules/jest-environment-node": {
"version": "29.7.0",
"resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-29.7.0.tgz",
@@ -8056,6 +8367,136 @@
"js-yaml": "bin/js-yaml.js"
}
},
+ "node_modules/jsdom": {
+ "version": "20.0.3",
+ "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-20.0.3.tgz",
+ "integrity": "sha512-SYhBvTh89tTfCD/CRdSOm13mOBa42iTaTyfyEWBdKcGdPxPtLFBXuHR8XHb33YNYaP+lLbmSvBTsnoesCNJEsQ==",
+ "dev": true,
+ "dependencies": {
+ "abab": "^2.0.6",
+ "acorn": "^8.8.1",
+ "acorn-globals": "^7.0.0",
+ "cssom": "^0.5.0",
+ "cssstyle": "^2.3.0",
+ "data-urls": "^3.0.2",
+ "decimal.js": "^10.4.2",
+ "domexception": "^4.0.0",
+ "escodegen": "^2.0.0",
+ "form-data": "^4.0.0",
+ "html-encoding-sniffer": "^3.0.0",
+ "http-proxy-agent": "^5.0.0",
+ "https-proxy-agent": "^5.0.1",
+ "is-potential-custom-element-name": "^1.0.1",
+ "nwsapi": "^2.2.2",
+ "parse5": "^7.1.1",
+ "saxes": "^6.0.0",
+ "symbol-tree": "^3.2.4",
+ "tough-cookie": "^4.1.2",
+ "w3c-xmlserializer": "^4.0.0",
+ "webidl-conversions": "^7.0.0",
+ "whatwg-encoding": "^2.0.0",
+ "whatwg-mimetype": "^3.0.0",
+ "whatwg-url": "^11.0.0",
+ "ws": "^8.11.0",
+ "xml-name-validator": "^4.0.0"
+ },
+ "engines": {
+ "node": ">=14"
+ },
+ "peerDependencies": {
+ "canvas": "^2.5.0"
+ },
+ "peerDependenciesMeta": {
+ "canvas": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/jsdom/node_modules/agent-base": {
+ "version": "6.0.2",
+ "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
+ "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
+ "dev": true,
+ "dependencies": {
+ "debug": "4"
+ },
+ "engines": {
+ "node": ">= 6.0.0"
+ }
+ },
+ "node_modules/jsdom/node_modules/http-proxy-agent": {
+ "version": "5.0.0",
+ "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz",
+ "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==",
+ "dev": true,
+ "dependencies": {
+ "@tootallnate/once": "2",
+ "agent-base": "6",
+ "debug": "4"
+ },
+ "engines": {
+ "node": ">= 6"
+ }
+ },
+ "node_modules/jsdom/node_modules/https-proxy-agent": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
+ "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
+ "dev": true,
+ "dependencies": {
+ "agent-base": "6",
+ "debug": "4"
+ },
+ "engines": {
+ "node": ">= 6"
+ }
+ },
+ "node_modules/jsdom/node_modules/parse5": {
+ "version": "7.2.0",
+ "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.2.0.tgz",
+ "integrity": "sha512-ZkDsAOcxsUMZ4Lz5fVciOehNcJ+Gb8gTzcA4yl3wnc273BAybYWrQ+Ks/OjCjSEpjvQkDSeZbybK9qj2VHHdGA==",
+ "dev": true,
+ "dependencies": {
+ "entities": "^4.5.0"
+ },
+ "funding": {
+ "url": "https://github.com/inikulin/parse5?sponsor=1"
+ }
+ },
+ "node_modules/jsdom/node_modules/tr46": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz",
+ "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==",
+ "dev": true,
+ "dependencies": {
+ "punycode": "^2.1.1"
+ },
+ "engines": {
+ "node": ">=12"
+ }
+ },
+ "node_modules/jsdom/node_modules/webidl-conversions": {
+ "version": "7.0.0",
+ "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
+ "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",
+ "dev": true,
+ "engines": {
+ "node": ">=12"
+ }
+ },
+ "node_modules/jsdom/node_modules/whatwg-url": {
+ "version": "11.0.0",
+ "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz",
+ "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==",
+ "dev": true,
+ "dependencies": {
+ "tr46": "^3.0.0",
+ "webidl-conversions": "^7.0.0"
+ },
+ "engines": {
+ "node": ">=12"
+ }
+ },
"node_modules/jsesc": {
"version": "2.5.2",
"resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz",
@@ -8502,7 +8943,6 @@
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
"dev": true,
- "peer": true,
"dependencies": {
"mime-db": "1.52.0"
},
@@ -8515,7 +8955,6 @@
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
"integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
"dev": true,
- "peer": true,
"engines": {
"node": ">= 0.6"
}
@@ -11216,6 +11655,12 @@
"inBundle": true,
"license": "ISC"
},
+ "node_modules/nwsapi": {
+ "version": "2.2.13",
+ "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.13.tgz",
+ "integrity": "sha512-cTGB9ptp9dY9A5VbMSe7fQBcl/tt22Vcqdq8+eN93rblOuE0aCFu4aZ2vMwct/2t+lFnosm8RkQW1I0Omb1UtQ==",
+ "dev": true
+ },
"node_modules/object-assign": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
@@ -11781,6 +12226,12 @@
"integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==",
"dev": true
},
+ "node_modules/psl": {
+ "version": "1.9.0",
+ "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz",
+ "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==",
+ "dev": true
+ },
"node_modules/punycode": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
@@ -11806,6 +12257,12 @@
}
]
},
+ "node_modules/querystringify": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz",
+ "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==",
+ "dev": true
+ },
"node_modules/queue-microtask": {
"version": "1.2.3",
"resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
@@ -11994,6 +12451,12 @@
"node": ">=0.10.0"
}
},
+ "node_modules/requires-port": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
+ "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==",
+ "dev": true
+ },
"node_modules/resolve": {
"version": "1.22.8",
"resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz",
@@ -12116,6 +12579,24 @@
"dev": true,
"peer": true
},
+ "node_modules/safer-buffer": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
+ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
+ "dev": true
+ },
+ "node_modules/saxes": {
+ "version": "6.0.0",
+ "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz",
+ "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==",
+ "dev": true,
+ "dependencies": {
+ "xmlchars": "^2.2.0"
+ },
+ "engines": {
+ "node": ">=v12.22.7"
+ }
+ },
"node_modules/schema-utils": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz",
@@ -12960,6 +13441,12 @@
"webpack": ">=2"
}
},
+ "node_modules/symbol-tree": {
+ "version": "3.2.4",
+ "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
+ "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==",
+ "dev": true
+ },
"node_modules/tapable": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz",
@@ -13268,6 +13755,30 @@
"node": ">=8.0"
}
},
+ "node_modules/tough-cookie": {
+ "version": "4.1.4",
+ "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.4.tgz",
+ "integrity": "sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag==",
+ "dev": true,
+ "dependencies": {
+ "psl": "^1.1.33",
+ "punycode": "^2.1.1",
+ "universalify": "^0.2.0",
+ "url-parse": "^1.5.3"
+ },
+ "engines": {
+ "node": ">=6"
+ }
+ },
+ "node_modules/tough-cookie/node_modules/universalify": {
+ "version": "0.2.0",
+ "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz",
+ "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==",
+ "dev": true,
+ "engines": {
+ "node": ">= 4.0.0"
+ }
+ },
"node_modules/tr46": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz",
@@ -13563,6 +14074,16 @@
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
}
},
+ "node_modules/url-parse": {
+ "version": "1.5.10",
+ "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
+ "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
+ "dev": true,
+ "dependencies": {
+ "querystringify": "^2.1.1",
+ "requires-port": "^1.0.0"
+ }
+ },
"node_modules/util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
@@ -13594,6 +14115,18 @@
"spdx-expression-parse": "^3.0.0"
}
},
+ "node_modules/w3c-xmlserializer": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-4.0.0.tgz",
+ "integrity": "sha512-d+BFHzbiCx6zGfz0HyQ6Rg69w9k19nviJspaj4yNscGjrHu94sVP+aRm75yEbCh+r2/yR+7q6hux9LVtbuTGBw==",
+ "dev": true,
+ "dependencies": {
+ "xml-name-validator": "^4.0.0"
+ },
+ "engines": {
+ "node": ">=14"
+ }
+ },
"node_modules/walker": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz",
@@ -13617,6 +14150,15 @@
"node": ">=10.13.0"
}
},
+ "node_modules/web-streams-polyfill": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0.tgz",
+ "integrity": "sha512-0zJXHRAYEjM2tUfZ2DiSOHAa2aw1tisnnhU3ufD57R8iefL+DcdJyRBRyJpG+NUimDgbTI/lH+gAE1PAvV3Cgw==",
+ "dev": true,
+ "engines": {
+ "node": ">= 8"
+ }
+ },
"node_modules/webidl-conversions": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz",
@@ -13711,6 +14253,27 @@
"dev": true,
"peer": true
},
+ "node_modules/whatwg-encoding": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz",
+ "integrity": "sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==",
+ "dev": true,
+ "dependencies": {
+ "iconv-lite": "0.6.3"
+ },
+ "engines": {
+ "node": ">=12"
+ }
+ },
+ "node_modules/whatwg-mimetype": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz",
+ "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==",
+ "dev": true,
+ "engines": {
+ "node": ">=12"
+ }
+ },
"node_modules/whatwg-url": {
"version": "7.1.0",
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz",
@@ -13873,6 +14436,42 @@
"node": "^12.13.0 || ^14.15.0 || >=16.0.0"
}
},
+ "node_modules/ws": {
+ "version": "8.18.0",
+ "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz",
+ "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==",
+ "dev": true,
+ "engines": {
+ "node": ">=10.0.0"
+ },
+ "peerDependencies": {
+ "bufferutil": "^4.0.1",
+ "utf-8-validate": ">=5.0.2"
+ },
+ "peerDependenciesMeta": {
+ "bufferutil": {
+ "optional": true
+ },
+ "utf-8-validate": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/xml-name-validator": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-4.0.0.tgz",
+ "integrity": "sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==",
+ "dev": true,
+ "engines": {
+ "node": ">=12"
+ }
+ },
+ "node_modules/xmlchars": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",
+ "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==",
+ "dev": true
+ },
"node_modules/xtend": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
diff --git a/package.json b/package.json
index 931b907..ea7da96 100644
--- a/package.json
+++ b/package.json
@@ -85,13 +85,16 @@
"@swc/core": "^1.7.26",
"@swc/jest": "^0.2.36",
"@types/jest": "^29.5.13",
+ "blob-polyfill": "^9.0.20240710",
"eslint": "^9.11.1",
"husky": "^9.1.6",
"jest": "^29.7.0",
+ "jest-environment-jsdom": "^29.7.0",
"prettier": "^3.3.3",
"swc-loader": "^0.2.6",
"tsup": "^8.3.0",
"typescript": "^5.6.2",
- "typescript-eslint": "^8.7.0"
+ "typescript-eslint": "^8.7.0",
+ "web-streams-polyfill": "^4.0.0"
}
}
diff --git a/src/index.ts b/src/index.ts
index 798227b..7fd2e25 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,7 +1,7 @@
export { getObjectDiff } from "./lib/object-diff";
export { getListDiff } from "./lib/list-diff";
export { isEqual, isObject } from "./lib/utils";
-export { streamListDiff } from "./lib/stream-list-diff";
+export * from "./lib/stream-list-diff";
export * from "./models/list";
export * from "./models/object";
export * from "./models/stream";
diff --git a/src/lib/stream-list-diff/client/index.ts b/src/lib/stream-list-diff/client/index.ts
new file mode 100644
index 0000000..76304a0
--- /dev/null
+++ b/src/lib/stream-list-diff/client/index.ts
@@ -0,0 +1,289 @@
+import { isClient } from "@lib/utils";
+import {
+ DataBuffer,
+ DEFAULT_LIST_STREAM_OPTIONS,
+ ListStreamOptions,
+ ReferenceProperty,
+} from "@models/stream";
+import { LIST_STATUS } from "@models/list";
+import {
+ Emitter,
+ EmitterEvents,
+ EventEmitter,
+ StreamListener,
+ StreamEvent,
+} from "../emitter";
+import { isDataValid, isValidChunkSize, outputDiffChunk } from "../utils";
+
+async function getDiffChunks<T extends Record<string, unknown>>(
+ prevStream: ReadableStream<T>,
+ nextStream: ReadableStream<T>,
+ referenceProperty: ReferenceProperty<T>,
+ emitter: Emitter<T>,
+ options: ListStreamOptions = DEFAULT_LIST_STREAM_OPTIONS,
+): Promise<void> {
+ if (!isValidChunkSize(options?.chunksSize)) {
+ return emitter.emit(
+ StreamEvent.Error,
+ new Error(
+ `The chunk size can't be negative. You entered the value '${options.chunksSize}'`,
+ ),
+ );
+ }
+
+ const prevList = prevStream.getReader();
+ const nextList = nextStream.getReader();
+ const { handleDiffChunk, releaseLastChunks } = outputDiffChunk<T>(emitter);
+ const prevDataBuffer: DataBuffer<T> = new Map();
+ const nextDataBuffer: DataBuffer<T> = new Map();
+ let currentPrevIndex = 0;
+ let currentNextIndex = 0;
+
+ async function processPrevStreamChunk(chunk: T) {
+ const { isValid, message } = isDataValid(
+ chunk,
+ referenceProperty,
+ "prevList",
+ );
+ if (!isValid) {
+ emitter.emit(StreamEvent.Error, new Error(message));
+ emitter.emit(StreamEvent.Finish);
+ return;
+ }
+ const ref = chunk[referenceProperty] as ReferenceProperty<T>;
+ const relatedChunk = nextDataBuffer.get(ref);
+
+ if (relatedChunk) {
+ nextDataBuffer.delete(ref);
+ const isDataEqual =
+ JSON.stringify(chunk) === JSON.stringify(relatedChunk.data);
+ const indexDiff = (relatedChunk.index as number) - currentPrevIndex;
+ if (isDataEqual) {
+ handleDiffChunk(
+ {
+ previousValue: chunk,
+ currentValue: relatedChunk.data,
+ prevIndex: currentPrevIndex,
+ newIndex: relatedChunk.index,
+ indexDiff,
+ status:
+ indexDiff === 0
+ ? LIST_STATUS.EQUAL
+ : options.considerMoveAsUpdate
+ ? LIST_STATUS.UPDATED
+ : LIST_STATUS.MOVED,
+ },
+ options,
+ );
+ } else {
+ handleDiffChunk(
+ {
+ previousValue: chunk,
+ currentValue: relatedChunk.data,
+ prevIndex: currentPrevIndex,
+ newIndex: relatedChunk.index,
+ indexDiff,
+ status: LIST_STATUS.UPDATED,
+ },
+ options,
+ );
+ }
+ } else {
+ prevDataBuffer.set(ref, { data: chunk, index: currentPrevIndex });
+ }
+ currentPrevIndex++;
+ }
+
+ async function processNextStreamChunk(chunk: T) {
+ const { isValid, message } = isDataValid(
+ chunk,
+ referenceProperty,
+ "nextList",
+ );
+ if (!isValid) {
+ emitter.emit(StreamEvent.Error, new Error(message));
+ emitter.emit(StreamEvent.Finish);
+ return;
+ }
+ const ref = chunk[referenceProperty] as ReferenceProperty<T>;
+ const relatedChunk = prevDataBuffer.get(ref);
+
+ if (relatedChunk) {
+ prevDataBuffer.delete(ref);
+ const isDataEqual =
+ JSON.stringify(chunk) === JSON.stringify(relatedChunk.data);
+ const indexDiff = currentNextIndex - (relatedChunk.index as number);
+ if (isDataEqual) {
+ handleDiffChunk(
+ {
+ previousValue: relatedChunk.data,
+ currentValue: chunk,
+ prevIndex: relatedChunk.index,
+ newIndex: currentNextIndex,
+ indexDiff,
+ status:
+ indexDiff === 0
+ ? LIST_STATUS.EQUAL
+ : options.considerMoveAsUpdate
+ ? LIST_STATUS.UPDATED
+ : LIST_STATUS.MOVED,
+ },
+ options,
+ );
+ } else {
+ handleDiffChunk(
+ {
+ previousValue: relatedChunk.data,
+ currentValue: chunk,
+ prevIndex: relatedChunk.index,
+ newIndex: currentNextIndex,
+ indexDiff,
+ status: LIST_STATUS.UPDATED,
+ },
+ options,
+ );
+ }
+ } else {
+ nextDataBuffer.set(ref, { data: chunk, index: currentNextIndex });
+ }
+ currentNextIndex++;
+ }
+
+ const readStream = async (
+ reader: ReadableStreamDefaultReader<T>,
+ processChunk: (chunk: T) => Promise<void>,
+ ) => {
+ let result;
+ while (!(result = await reader.read()).done) {
+ await processChunk(result.value);
+ }
+ };
+
+ await Promise.all([
+ readStream(prevList, async (chunk) => {
+ await processPrevStreamChunk(chunk);
+ }),
+ readStream(nextList, async (chunk) => {
+ await processNextStreamChunk(chunk);
+ }),
+ ]);
+
+ for (const [key, chunk] of prevDataBuffer.entries()) {
+ handleDiffChunk(
+ {
+ previousValue: chunk.data,
+ currentValue: null,
+ prevIndex: chunk.index,
+ newIndex: null,
+ indexDiff: null,
+ status: LIST_STATUS.DELETED,
+ },
+ options,
+ );
+ prevDataBuffer.delete(key);
+ }
+ for (const [key, chunk] of nextDataBuffer.entries()) {
+ handleDiffChunk(
+ {
+ previousValue: null,
+ currentValue: chunk.data,
+ prevIndex: null,
+ newIndex: chunk.index,
+ indexDiff: null,
+ status: LIST_STATUS.ADDED,
+ },
+ options,
+ );
+ nextDataBuffer.delete(key);
+ }
+
+ releaseLastChunks();
+ return emitter.emit(StreamEvent.Finish);
+}
+
+async function getValidClientStream<T extends Record<string, unknown>>(
+ input: ReadableStream<T> | T[] | File,
+ listType: "prevList" | "nextList",
+): Promise<ReadableStream<T>> {
+ if (Array.isArray(input)) {
+ return new ReadableStream({
+ start(controller) {
+ input.forEach((item) => controller.enqueue(item));
+ controller.close();
+ },
+ });
+ }
+
+ if (input instanceof ReadableStream) {
+ return input;
+ }
+
+ if (input instanceof File) {
+ const fileText = await input.text();
+ let jsonData: T[];
+ try {
+ jsonData = JSON.parse(fileText);
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
+ } catch (_: unknown) {
+ throw new Error(`Your ${listType} is not a valid JSON array.`);
+ }
+
+ if (!Array.isArray(jsonData)) {
+ throw new Error(`Your ${listType} is not a JSON array.`);
+ }
+ return new ReadableStream({
+ start(controller) {
+ jsonData.forEach((item) => controller.enqueue(item));
+ controller.close();
+ },
+ });
+ }
+
+ throw new Error(
+ `Invalid ${listType}. Expected ReadableStream, Array, or File.`,
+ );
+}
+
+/**
+ * Streams the diff of two object lists
+ * @param {Record<string, unknown>[]} prevList - The original object list.
+ * @param {Record<string, unknown>[]} nextList - The new object list.
+ * @param {ReferenceProperty<T>} referenceProperty - A common property in all the objects of your lists (e.g. `id`)
+ * @param {ListStreamOptions} options - Options to refine your output.
+ - `chunksSize`: the number of object diffs returned by each streamed chunk. (e.g. `0` = 1 object diff by chunk, `10` = 10 object diffs by chunk).
+ - `showOnly`: returns only the values whose status you are interested in. (e.g. `["added", "equal"]`)
+ - `considerMoveAsUpdate`: if set to `true` a `moved` object will be considered as `updated`
+ * @returns EventEmitter
+ */
+export function streamListDiffClient<T extends Record<string, unknown>>(
+ prevList: ReadableStream<T> | File | T[],
+ nextList: ReadableStream<T> | File | T[],
+ referenceProperty: ReferenceProperty<T>,
+ options: ListStreamOptions = DEFAULT_LIST_STREAM_OPTIONS,
+): StreamListener<T> {
+ if (!isClient()) {
+ throw new Error(
+ "streamListDiffClient can only be used in a browser environment. Please use streamListDiff instead.",
+ );
+ }
+ const emitter = new EventEmitter<EmitterEvents<T>>();
+ setTimeout(async () => {
+ try {
+ const [prevStream, nextStream] = await Promise.all([
+ getValidClientStream(prevList, "prevList"),
+ getValidClientStream(nextList, "nextList"),
+ ]);
+
+ getDiffChunks(
+ prevStream,
+ nextStream,
+ referenceProperty,
+ emitter,
+ options,
+ );
+ } catch (err) {
+ return emitter.emit(StreamEvent.Error, err as Error);
+ }
+ }, 0);
+ return emitter as StreamListener<T>;
+}
diff --git a/src/lib/stream-list-diff/client/stream-list-diff-client.test.ts b/src/lib/stream-list-diff/client/stream-list-diff-client.test.ts
new file mode 100644
index 0000000..bc39cf9
--- /dev/null
+++ b/src/lib/stream-list-diff/client/stream-list-diff-client.test.ts
@@ -0,0 +1,1062 @@
+/**
+ * @jest-environment jsdom
+ */
+import "blob-polyfill";
+import { ReadableStream } from "web-streams-polyfill";
+import { LIST_STATUS } from "@models/list";
+import { StreamListDiff } from "@models/stream";
+import { streamListDiffClient } from ".";
+import prevListFile from "../../../mocks/prevList.json";
+import nextListFile from "../../../mocks/nextList.json";
+
+//@ts-expect-error - the ReadableStream polyfill is necessary to test ReadableStream in a Node environment.
+global.ReadableStream = ReadableStream;
+
+describe("data emission", () => {
+ it("emits 'data' event and consider the all the nextList added if no prevList is provided", (done) => {
+ const nextList = [
+ { id: 1, name: "Item 1" },
+ { id: 2, name: "Item 2" },
+ ];
+ const diff = streamListDiffClient([], nextList, "id", { chunksSize: 2 });
+
+ const expectedChunks = [
+ {
+ previousValue: null,
+ currentValue: { id: 1, name: "Item 1" },
+ prevIndex: null,
+ newIndex: 0,
+ indexDiff: null,
+ status: LIST_STATUS.ADDED,
+ },
+ {
+ previousValue: null,
+ currentValue: { id: 2, name: "Item 2" },
+ prevIndex: null,
+ newIndex: 1,
+ indexDiff: null,
+ status: LIST_STATUS.ADDED,
+ },
+ ];
+ let chunkCount = 0;
+ diff.on("data", (chunk) => {
+ expect(chunk).toStrictEqual(expectedChunks);
+ chunkCount++;
+ });
+ diff.on("finish", () => {
+ expect(chunkCount).toBe(1);
+ done();
+ });
+ });
+ it("emits 'data' event and consider the all the prevList deleted if no nextList is provided", (done) => {
+ const prevList = [
+ { id: 1, name: "Item 1" },
+ { id: 2, name: "Item 2" },
+ ];
+ const diff = streamListDiffClient(prevList, [], "id", { chunksSize: 2 });
+
+ const expectedChunks = [
+ {
+ previousValue: { id: 1, name: "Item 1" },
+ currentValue: null,
+ prevIndex: 0,
+ newIndex: null,
+ indexDiff: null,
+ status: LIST_STATUS.DELETED,
+ },
+ {
+ previousValue: { id: 2, name: "Item 2" },
+ currentValue: null,
+ prevIndex: 1,
+ newIndex: null,
+ indexDiff: null,
+ status: LIST_STATUS.DELETED,
+ },
+ ];
+ let chunkCount = 0;
+ diff.on("data", (chunk) => {
+ expect(chunk).toStrictEqual(expectedChunks);
+ chunkCount++;
+ });
+ diff.on("error", (err) => console.error("shiiiite", err));
+ diff.on("finish", () => {
+ expect(chunkCount).toBe(1);
+ done();
+ });
+ });
+ it("emits 'data' event with one object diff by chunk if chunkSize is 0 or undefined", (done) => {
+ const prevList = [
+ { id: 1, name: "Item 1" },
+ { id: 2, name: "Item 2" },
+ ];
+ const nextList = [
+ { id: 2, name: "Item 2" },
+ { id: 3, name: "Item 3" },
+ ];
+ const diff = streamListDiffClient(prevList, nextList, "id");
+
+ const expectedChunks = [
+ [
+ {
+ previousValue: { id: 2, name: "Item 2" },
+ currentValue: { id: 2, name: "Item 2" },
+ prevIndex: 1,
+ newIndex: 0,
+ indexDiff: -1,
+ status: LIST_STATUS.MOVED,
+ },
+ ],
+ [
+ {
+ previousValue: { id: 1, name: "Item 1" },
+ currentValue: null,
+ prevIndex: 0,
+ newIndex: null,
+ indexDiff: null,
+ status: LIST_STATUS.DELETED,
+ },
+ ],
+ [
+ {
+ previousValue: null,
+ currentValue: { id: 3, name: "Item 3" },
+ prevIndex: null,
+ newIndex: 1,
+ indexDiff: null,
+ status: LIST_STATUS.ADDED,
+ },
+ ],
+ ];
+
+ let chunkCount = 0;
+
+ diff.on("data", (chunk) => {
+ expect(chunk).toStrictEqual(expectedChunks[chunkCount]);
+ chunkCount++;
+ });
+ diff.on("finish", () => {
+ expect(chunkCount).toBe(3);
+ done();
+ });
+ });
+ it("emits 'data' event with 5 object diff by chunk and return the last object diff in a one entry chunk at the end", (done) => {
+ const prevList = [
+ { id: 1, name: "Item 1" },
+ { id: 2, name: "Item 2" },
+ { id: 3, name: "Item 3" },
+ { id: 4, name: "Item 4" },
+ { id: 5, name: "Item 5" },
+ { id: 6, name: "Item 6" },
+ { id: 7, name: "Item 7" },
+ { id: 8, name: "Item 8" },
+ { id: 9, name: "Item 9" },
+ { id: 10, name: "Item 10" },
+ ];
+ const nextList = [
+ { id: 1, name: "Item 1" },
+ { id: 2, name: "Item Two" },
+ { id: 3, name: "Item 3" },
+ { id: 5, name: "Item 5" },
+ { id: 6, name: "Item Six" },
+ { id: 7, name: "Item 7" },
+ { id: 10, name: "Item 10" },
+ { id: 11, name: "Item 11" },
+ { id: 9, name: "Item 9" },
+ { id: 8, name: "Item 8" },
+ ];
+ const diff = streamListDiffClient(prevList, nextList, "id", {
+ chunksSize: 5,
+ });
+
+ const expectedChunks = [
+ [
+ {
+ previousValue: { id: 1, name: "Item 1" },
+ currentValue: { id: 1, name: "Item 1" },
+ prevIndex: 0,
+ newIndex: 0,
+ indexDiff: 0,
+ status: LIST_STATUS.EQUAL,
+ },
+ {
+ previousValue: { id: 2, name: "Item 2" },
+ currentValue: { id: 2, name: "Item Two" },
+ prevIndex: 1,
+ newIndex: 1,
+ indexDiff: 0,
+ status: LIST_STATUS.UPDATED,
+ },
+ {
+ previousValue: { id: 3, name: "Item 3" },
+ currentValue: { id: 3, name: "Item 3" },
+ prevIndex: 2,
+ newIndex: 2,
+ indexDiff: 0,
+ status: LIST_STATUS.EQUAL,
+ },
+ {
+ previousValue: { id: 5, name: "Item 5" },
+ currentValue: { id: 5, name: "Item 5" },
+ prevIndex: 4,
+ newIndex: 3,
+ indexDiff: -1,
+ status: LIST_STATUS.MOVED,
+ },
+ {
+ previousValue: { id: 6, name: "Item 6" },
+ currentValue: { id: 6, name: "Item Six" },
+ prevIndex: 5,
+ newIndex: 4,
+ indexDiff: -1,
+ status: LIST_STATUS.UPDATED,
+ },
+ ],
+ [
+ {
+ previousValue: { id: 7, name: "Item 7" },
+ currentValue: { id: 7, name: "Item 7" },
+ prevIndex: 6,
+ newIndex: 5,
+ indexDiff: -1,
+ status: LIST_STATUS.MOVED,
+ },
+ {
+ previousValue: { id: 9, name: "Item 9" },
+ currentValue: { id: 9, name: "Item 9" },
+ prevIndex: 8,
+ newIndex: 8,
+ indexDiff: 0,
+ status: LIST_STATUS.EQUAL,
+ },
+ {
+ previousValue: { id: 10, name: "Item 10" },
+ currentValue: { id: 10, name: "Item 10" },
+ prevIndex: 9,
+ newIndex: 6,
+ indexDiff: -3,
+ status: LIST_STATUS.MOVED,
+ },
+ {
+ previousValue: { id: 8, name: "Item 8" },
+ currentValue: { id: 8, name: "Item 8" },
+ prevIndex: 7,
+ newIndex: 9,
+ indexDiff: 2,
+ status: LIST_STATUS.MOVED,
+ },
+ {
+ previousValue: { id: 4, name: "Item 4" },
+ currentValue: null,
+ prevIndex: 3,
+ newIndex: null,
+ indexDiff: null,
+ status: LIST_STATUS.DELETED,
+ },
+ ],
+ [
+ {
+ previousValue: null,
+ currentValue: { id: 11, name: "Item 11" },
+ prevIndex: null,
+ newIndex: 7,
+ indexDiff: null,
+ status: LIST_STATUS.ADDED,
+ },
+ ],
+ ];
+
+ let chunkCount = 0;
+
+ diff.on("data", (chunk) => {
+ expect(chunk).toStrictEqual(expectedChunks[chunkCount]);
+ chunkCount++;
+ });
+
+ diff.on("finish", () => {
+ expect(chunkCount).toBe(3);
+ done();
+ });
+ });
+ it("emits 'data' event with all the objects diff in a single chunk if the chunkSize is bigger than the provided lists ", (done) => {
+ const prevList = [
+ { id: 1, name: "Item 1" },
+ { id: 2, name: "Item 2" },
+ { id: 3, name: "Item 3" },
+ { id: 4, name: "Item 4" },
+ ];
+ const nextList = [
+ { id: 1, name: "Item 1" },
+ { id: 2, name: "Item Two" },
+ { id: 3, name: "Item 3" },
+ { id: 5, name: "Item 5" },
+ ];
+
+ const diff = streamListDiffClient(prevList, nextList, "id", {
+ chunksSize: 150,
+ });
+
+ const expectedChunks = [
+ {
+ previousValue: { id: 1, name: "Item 1" },
+ currentValue: { id: 1, name: "Item 1" },
+ prevIndex: 0,
+ newIndex: 0,
+ indexDiff: 0,
+ status: LIST_STATUS.EQUAL,
+ },
+ {
+ previousValue: { id: 2, name: "Item 2" },
+ currentValue: { id: 2, name: "Item Two" },
+ prevIndex: 1,
+ newIndex: 1,
+ indexDiff: 0,
+ status: LIST_STATUS.UPDATED,
+ },
+ {
+ previousValue: { id: 3, name: "Item 3" },
+ currentValue: { id: 3, name: "Item 3" },
+ prevIndex: 2,
+ newIndex: 2,
+ indexDiff: 0,
+ status: LIST_STATUS.EQUAL,
+ },
+ {
+ previousValue: { id: 4, name: "Item 4" },
+ currentValue: null,
+ prevIndex: 3,
+ newIndex: null,
+ indexDiff: null,
+ status: LIST_STATUS.DELETED,
+ },
+ {
+ previousValue: null,
+ currentValue: { id: 5, name: "Item 5" },
+ prevIndex: null,
+ newIndex: 3,
+ indexDiff: null,
+ status: LIST_STATUS.ADDED,
+ },
+ ];
+
+ let chunkCount = 0;
+ diff.on("data", (chunk) => {
+ expect(chunk).toStrictEqual(expectedChunks);
+ chunkCount++;
+ });
+
+ diff.on("finish", () => {
+ expect(chunkCount).toBe(1);
+ done();
+ });
+ });
+ it("emits 'data' event with moved objects considered as updated if considerMoveAsUpdate is true", (done) => {
+ const prevList = [
+ { id: 1, name: "Item 1" },
+ { id: 2, name: "Item 2" },
+ { id: 3, name: "Item 3" },
+ { id: 4, name: "Item 4" },
+ ];
+ const nextList = [
+ { id: 2, name: "Item Two" },
+ { id: 1, name: "Item 1" },
+ { id: 3, name: "Item 3" },
+ { id: 5, name: "Item 5" },
+ ];
+ const diff = streamListDiffClient(prevList, nextList, "id", {
+ chunksSize: 5,
+ considerMoveAsUpdate: true,
+ });
+
+ const expectedChunks = [
+ {
+ previousValue: { id: 2, name: "Item 2" },
+ currentValue: { id: 2, name: "Item Two" },
+ prevIndex: 1,
+ newIndex: 0,
+ indexDiff: -1,
+ status: LIST_STATUS.UPDATED,
+ },
+ {
+ previousValue: { id: 1, name: "Item 1" },
+ currentValue: { id: 1, name: "Item 1" },
+ prevIndex: 0,
+ newIndex: 1,
+ indexDiff: 1,
+ status: LIST_STATUS.UPDATED,
+ },
+ {
+ previousValue: { id: 3, name: "Item 3" },
+ currentValue: { id: 3, name: "Item 3" },
+ prevIndex: 2,
+ newIndex: 2,
+ indexDiff: 0,
+ status: LIST_STATUS.EQUAL,
+ },
+ {
+ previousValue: { id: 4, name: "Item 4" },
+ currentValue: null,
+ prevIndex: 3,
+ newIndex: null,
+ indexDiff: null,
+ status: LIST_STATUS.DELETED,
+ },
+ {
+ previousValue: null,
+ currentValue: { id: 5, name: "Item 5" },
+ prevIndex: null,
+ newIndex: 3,
+ indexDiff: null,
+ status: LIST_STATUS.ADDED,
+ },
+ ];
+
+ let chunkCount = 0;
+ diff.on("data", (chunk) => {
+ expect(chunk).toStrictEqual(expectedChunks);
+ chunkCount++;
+ });
+
+ diff.on("finish", () => {
+ expect(chunkCount).toBe(1);
+ done();
+ });
+ });
+ it("emits 'data' event only with objects diff whose status match with showOnly's", (done) => {
+ const prevList = [
+ { id: 1, name: "Item 1" },
+ { id: 2, name: "Item 2" },
+ { id: 3, name: "Item 3" },
+ { id: 4, name: "Item 4" },
+ ];
+ const nextList = [
+ { id: 2, name: "Item Two" },
+ { id: 1, name: "Item 1" },
+ { id: 3, name: "Item 3" },
+ { id: 5, name: "Item 5" },
+ ];
+ const diff = streamListDiffClient(prevList, nextList, "id", {
+ chunksSize: 5,
+ showOnly: ["added", "deleted"],
+ });
+
+ const expectedChunks = [
+ {
+ previousValue: { id: 4, name: "Item 4" },
+ currentValue: null,
+ prevIndex: 3,
+ newIndex: null,
+ indexDiff: null,
+ status: LIST_STATUS.DELETED,
+ },
+ {
+ previousValue: null,
+ currentValue: { id: 5, name: "Item 5" },
+ prevIndex: null,
+ newIndex: 3,
+ indexDiff: null,
+ status: LIST_STATUS.ADDED,
+ },
+ ];
+
+ let chunkCount = 0;
+ diff.on("data", (chunk) => {
+ expect(chunk).toStrictEqual(expectedChunks);
+ chunkCount++;
+ });
+
+ diff.on("finish", () => {
+ expect(chunkCount).toBe(1);
+ done();
+ });
+ });
+ it("emits 'data' event with deep nested objects diff", (done) => {
+ const prevList = [
+ {
+ id: 1,
+ name: "Item 1",
+ user: { role: "admin", hobbies: ["golf", "football"] },
+ },
+ { id: 2, name: "Item 2" },
+ { id: 3, name: "Item 3", user: { role: "admin", hobbies: ["rugby"] } },
+ {
+ id: 4,
+ name: "Item 4",
+ user: { role: "reader", hobbies: ["video games", "fishing"] },
+ },
+ { id: 5, name: "Item 5" },
+ { id: 6, name: "Item 6", user: { role: "root", hobbies: ["coding"] } },
+ { id: 7, name: "Item 7" },
+ { id: 8, name: "Item 8" },
+ { id: 9, name: "Item 9" },
+ {
+ id: 10,
+ name: "Item 10",
+ user: {
+ role: "root",
+ hobbies: ["coding"],
+ skills: { driving: true, diving: false },
+ },
+ },
+ ];
+ const nextList = [
+ {
+ id: 1,
+ name: "Item 1",
+ user: { role: "admin", hobbies: ["golf", "football"] },
+ },
+ { id: 2, name: "Item Two" },
+ { id: 3, name: "Item 3", user: { role: "admin", hobbies: ["rugby"] } },
+ { id: 5, name: "Item 5" },
+ { id: 6, name: "Item 6", user: { role: "root", hobbies: ["farming"] } },
+ { id: 7, name: "Item 7" },
+ {
+ id: 10,
+ name: "Item 10",
+ user: {
+ role: "root",
+ hobbies: ["coding"],
+ skills: { driving: true, diving: false },
+ },
+ },
+ { id: 11, name: "Item 11" },
+ { id: 9, name: "Item 9" },
+ { id: 8, name: "Item 8" },
+ ];
+ const diff = streamListDiffClient(prevList, nextList, "id", {
+ chunksSize: 5,
+ });
+
+ const expectedChunks = [
+ [
+ {
+ previousValue: {
+ id: 1,
+ name: "Item 1",
+ user: { role: "admin", hobbies: ["golf", "football"] },
+ },
+ currentValue: {
+ id: 1,
+ name: "Item 1",
+ user: { role: "admin", hobbies: ["golf", "football"] },
+ },
+ prevIndex: 0,
+ newIndex: 0,
+ indexDiff: 0,
+ status: LIST_STATUS.EQUAL,
+ },
+ {
+ previousValue: { id: 2, name: "Item 2" },
+ currentValue: { id: 2, name: "Item Two" },
+ prevIndex: 1,
+ newIndex: 1,
+ indexDiff: 0,
+ status: LIST_STATUS.UPDATED,
+ },
+ {
+ previousValue: {
+ id: 3,
+ name: "Item 3",
+ user: { role: "admin", hobbies: ["rugby"] },
+ },
+ currentValue: {
+ id: 3,
+ name: "Item 3",
+ user: { role: "admin", hobbies: ["rugby"] },
+ },
+ prevIndex: 2,
+ newIndex: 2,
+ indexDiff: 0,
+ status: LIST_STATUS.EQUAL,
+ },
+ {
+ previousValue: { id: 5, name: "Item 5" },
+ currentValue: { id: 5, name: "Item 5" },
+ prevIndex: 4,
+ newIndex: 3,
+ indexDiff: -1,
+ status: LIST_STATUS.MOVED,
+ },
+ {
+ previousValue: {
+ id: 6,
+ name: "Item 6",
+ user: { role: "root", hobbies: ["coding"] },
+ },
+ currentValue: {
+ id: 6,
+ name: "Item 6",
+ user: { role: "root", hobbies: ["farming"] },
+ },
+ prevIndex: 5,
+ newIndex: 4,
+ indexDiff: -1,
+ status: LIST_STATUS.UPDATED,
+ },
+ ],
+ [
+ {
+ previousValue: { id: 7, name: "Item 7" },
+ currentValue: { id: 7, name: "Item 7" },
+ prevIndex: 6,
+ newIndex: 5,
+ indexDiff: -1,
+ status: LIST_STATUS.MOVED,
+ },
+ {
+ previousValue: { id: 9, name: "Item 9" },
+ currentValue: { id: 9, name: "Item 9" },
+ prevIndex: 8,
+ newIndex: 8,
+ indexDiff: 0,
+ status: LIST_STATUS.EQUAL,
+ },
+ {
+ previousValue: {
+ id: 10,
+ name: "Item 10",
+ user: {
+ role: "root",
+ hobbies: ["coding"],
+ skills: { driving: true, diving: false },
+ },
+ },
+ currentValue: {
+ id: 10,
+ name: "Item 10",
+ user: {
+ role: "root",
+ hobbies: ["coding"],
+ skills: { driving: true, diving: false },
+ },
+ },
+ prevIndex: 9,
+ newIndex: 6,
+ indexDiff: -3,
+ status: LIST_STATUS.MOVED,
+ },
+ {
+ previousValue: { id: 8, name: "Item 8" },
+ currentValue: { id: 8, name: "Item 8" },
+ prevIndex: 7,
+ newIndex: 9,
+ indexDiff: 2,
+ status: LIST_STATUS.MOVED,
+ },
+ {
+ previousValue: {
+ id: 4,
+ name: "Item 4",
+ user: { role: "reader", hobbies: ["video games", "fishing"] },
+ },
+ currentValue: null,
+ prevIndex: 3,
+ newIndex: null,
+ indexDiff: null,
+ status: LIST_STATUS.DELETED,
+ },
+ ],
+ [
+ {
+ previousValue: null,
+ currentValue: { id: 11, name: "Item 11" },
+ prevIndex: null,
+ newIndex: 7,
+ indexDiff: null,
+ status: LIST_STATUS.ADDED,
+ },
+ ],
+ ];
+
+ let chunkCount = 0;
+
+ diff.on("data", (chunk) => {
+ expect(chunk).toStrictEqual(expectedChunks[chunkCount]);
+ chunkCount++;
+ });
+
+ diff.on("finish", () => {
+ expect(chunkCount).toBe(3);
+ done();
+ });
+ });
+});
+
+describe("input handling", () => {
+ const prevList = [
+ { id: 1, name: "Item 1" },
+ { id: 2, name: "Item 2" },
+ { id: 3, name: "Item 3" },
+ { id: 4, name: "Item 4" },
+ ];
+ const nextList = [
+ { id: 1, name: "Item 1" },
+ { id: 2, name: "Item Two" },
+ { id: 3, name: "Item 3" },
+ { id: 5, name: "Item 5" },
+ ];
+ const expectedChunks = [
+ {
+ previousValue: { id: 1, name: "Item 1" },
+ currentValue: { id: 1, name: "Item 1" },
+ prevIndex: 0,
+ newIndex: 0,
+ indexDiff: 0,
+ status: LIST_STATUS.EQUAL,
+ },
+ {
+ previousValue: { id: 2, name: "Item 2" },
+ currentValue: { id: 2, name: "Item Two" },
+ prevIndex: 1,
+ newIndex: 1,
+ indexDiff: 0,
+ status: LIST_STATUS.UPDATED,
+ },
+ {
+ previousValue: { id: 3, name: "Item 3" },
+ currentValue: { id: 3, name: "Item 3" },
+ prevIndex: 2,
+ newIndex: 2,
+ indexDiff: 0,
+ status: LIST_STATUS.EQUAL,
+ },
+ {
+ previousValue: { id: 4, name: "Item 4" },
+ currentValue: null,
+ prevIndex: 3,
+ newIndex: null,
+ indexDiff: null,
+ status: LIST_STATUS.DELETED,
+ },
+ {
+ previousValue: null,
+ currentValue: { id: 5, name: "Item 5" },
+ prevIndex: null,
+ newIndex: 3,
+ indexDiff: null,
+ status: LIST_STATUS.ADDED,
+ },
+ ];
+
+ it("handles two readable streams", (done) => {
+ const prevStream = new ReadableStream({
+ start(controller) {
+ prevList.forEach((item) => controller.enqueue(item));
+ controller.close();
+ },
+ });
+ const nextStream = new ReadableStream({
+ start(controller) {
+ nextList.forEach((item) => controller.enqueue(item));
+ controller.close();
+ },
+ });
+
+ const diff = streamListDiffClient(prevStream, nextStream, "id", {
+ chunksSize: 5,
+ });
+
+ let chunkCount = 0;
+ diff.on("data", (chunk) => {
+ expect(chunk).toStrictEqual(expectedChunks);
+ chunkCount++;
+ });
+ diff.on("error", (err) => console.error("sheeeet", err));
+ diff.on("finish", () => {
+ expect(chunkCount).toBe(1);
+ done();
+ });
+ });
+ it("handles two local files", (done) => {
+ const prevFile = new File([JSON.stringify(prevListFile)], "prevList.json", {
+ type: "application/json",
+ });
+
+ const nextFile = new File([JSON.stringify(nextListFile)], "nextList.json", {
+ type: "application/json",
+ });
+
+ const diff = streamListDiffClient(prevFile, nextFile, "id", {
+ chunksSize: 5,
+ });
+
+ let chunkCount = 0;
+ diff.on("data", (chunk) => {
+ expect(chunk).toStrictEqual(expectedChunks);
+ chunkCount++;
+ });
+ diff.on("error", (err) => console.error("sheeeet", err));
+ diff.on("finish", () => {
+ expect(chunkCount).toBe(1);
+ done();
+ });
+ });
+ it("handles a readable stream against a local file", (done) => {
+ const prevStream = new ReadableStream({
+ start(controller) {
+ prevList.forEach((item) => controller.enqueue(item));
+ controller.close();
+ },
+ });
+ const nextFile = new File([JSON.stringify(nextListFile)], "nextList.json", {
+ type: "application/json",
+ });
+
+ const diff = streamListDiffClient(prevStream, nextFile, "id", {
+ chunksSize: 5,
+ });
+
+ let chunkCount = 0;
+ diff.on("data", (chunk) => {
+ expect(chunk).toStrictEqual(expectedChunks);
+ chunkCount++;
+ });
+ diff.on("error", (err) => console.error("sheeeet", err));
+ diff.on("finish", () => {
+ expect(chunkCount).toBe(1);
+ done();
+ });
+ });
+ it("handles a readable stream against an array", (done) => {
+ const prevStream = new ReadableStream({
+ start(controller) {
+ prevList.forEach((item) => controller.enqueue(item));
+ controller.close();
+ },
+ });
+
+ const diff = streamListDiffClient(prevStream, nextList, "id", {
+ chunksSize: 5,
+ });
+
+ let chunkCount = 0;
+ diff.on("data", (chunk) => {
+ expect(chunk).toStrictEqual(expectedChunks);
+ chunkCount++;
+ });
+ diff.on("error", (err) => console.error("sheeeet", err));
+ diff.on("finish", () => {
+ expect(chunkCount).toBe(1);
+ done();
+ });
+ });
+ it("handles a local file against an array", (done) => {
+ const prevFile = new File([JSON.stringify(prevListFile)], "prevList.json", {
+ type: "application/json",
+ });
+
+ const diff = streamListDiffClient(prevFile, nextList, "id", {
+ chunksSize: 5,
+ });
+
+ let chunkCount = 0;
+ diff.on("data", (chunk) => {
+ expect(chunk).toStrictEqual(expectedChunks);
+ chunkCount++;
+ });
+ diff.on("error", (err) => console.error("sheeeet", err));
+ diff.on("finish", () => {
+ expect(chunkCount).toBe(1);
+ done();
+ });
+ });
+});
+
+describe("finish event", () => {
+ it("emits 'finish' event if no prevList nor nextList is provided", (done) => {
+ const diff = streamListDiffClient([], [], "id");
+ diff.on("finish", () => done());
+ });
+ it("emits 'finish' event when all the chunks have been processed", (done) => {
+ const prevList = [
+ { id: 1, name: "Item 1" },
+ { id: 2, name: "Item 2" },
+ ];
+ const nextList = [
+ { id: 2, name: "Item 2" },
+ { id: 3, name: "Item 3" },
+ ];
+ const diff = streamListDiffClient(prevList, nextList, "id");
+ diff.on("finish", () => done());
+ });
+});
+
+describe("error event", () => {
+ test("emits 'error' event when prevList has invalid data", (done) => {
+ const prevList = [
+ { id: 1, name: "Item 1" },
+ "hello",
+ { id: 2, name: "Item 2" },
+ ];
+ const nextList = [
+ { id: 1, name: "Item 1" },
+ { id: 2, name: "Item 2" },
+ ];
+
+ // @ts-expect-error prevList is invalid by design for the test
+ const diff = streamListDiffClient(prevList, nextList, "id");
+
+ diff.on("error", (err) => {
+ expect(err["message"]).toEqual(
+ `Your prevList must only contain valid objects. Found 'hello'`,
+ );
+ done();
+ });
+ });
+
+ test("emits 'error' event when nextList has invalid data", (done) => {
+ const prevList = [
+ { id: 1, name: "Item 1" },
+ { id: 2, name: "Item 2" },
+ ];
+ const nextList = [
+ { id: 1, name: "Item 1" },
+ "hello",
+ { id: 2, name: "Item 2" },
+ ];
+
+ // @ts-expect-error nextList is invalid by design for the test
+ const diff = streamListDiffClient(prevList, nextList, "id");
+
+ diff.on("error", (err) => {
+ expect(err["message"]).toEqual(
+ `Your nextList must only contain valid objects. Found 'hello'`,
+ );
+ done();
+ });
+ });
+
+ test("emits 'error' event when all prevList ojects don't have the requested reference property", (done) => {
+ const prevList = [{ id: 1, name: "Item 1" }, { name: "Item 2" }];
+ const nextList = [
+ { id: 1, name: "Item 1" },
+ { id: 2, name: "Item 2" },
+ ];
+
+ const diff = streamListDiffClient(prevList, nextList, "id");
+
+ diff.on("error", (err) => {
+ expect(err["message"]).toEqual(
+ `The reference property 'id' is not available in all the objects of your prevList.`,
+ );
+ done();
+ });
+ });
+
+ test("emits 'error' event when all nextList ojects don't have the requested reference property", (done) => {
+ const prevList = [
+ { id: 1, name: "Item 1" },
+ { id: 2, name: "Item 2" },
+ ];
+ const nextList = [{ id: 1, name: "Item 1" }, { name: "Item 2" }];
+
+ const diff = streamListDiffClient(prevList, nextList, "id");
+
+ diff.on("error", (err) => {
+ expect(err["message"]).toEqual(
+ `The reference property 'id' is not available in all the objects of your nextList.`,
+ );
+ done();
+ });
+ });
+
+ test("emits 'error' event when the chunkSize option is negative", (done) => {
+ const prevList = [
+ { id: 1, name: "Item 1" },
+ { id: 2, name: "Item 2" },
+ ];
+ const nextList = [{ id: 1, name: "Item 1" }, { name: "Item 2" }];
+
+ const diff = streamListDiffClient(prevList, nextList, "id", {
+ chunksSize: -3,
+ });
+
+ diff.on("error", (err) => {
+ expect(err["message"]).toEqual(
+ "The chunk size can't be negative. You entered the value '-3'",
+ );
+ done();
+ });
+ });
+
+ test("emits 'error' event when the prevList is not a valid type", (done) => {
+ const nextList = [{ id: 1, name: "Item 1" }, { name: "Item 2" }];
+
+ // @ts-expect-error - prevList is invalid by design for the test
+ const diff = streamListDiffClient({ name: "hello" }, nextList, "id");
+
+ diff.on("error", (err) => {
+ expect(err["message"]).toEqual(
+ "Invalid prevList. Expected ReadableStream, Array, or File.",
+ );
+ done();
+ });
+ });
+ test("emits 'error' event when the nextList is not a valid type", (done) => {
+ const prevList = [{ id: 1, name: "Item 1" }, { name: "Item 2" }];
+
+ // @ts-expect-error - nextList is invalid by design for the test
+ const diff = streamListDiffClient(prevList, null, "id");
+
+ diff.on("error", (err) => {
+ expect(err["message"]).toEqual(
+ "Invalid nextList. Expected ReadableStream, Array, or File.",
+ );
+ done();
+ });
+ });
+});
+
+const generateLargeDataset = (count: number) => {
+ const data: Array<{ id: number; value: string }> = [];
+ for (let i = 0; i < count; i++) {
+ data.push({ id: i, value: `value-${i}` });
+ }
+ return data;
+};
+
+describe("performance", () => {
+ it("process 100.000 in each stream", (done) => {
+ const numEntries = 100_000;
+
+ const prevList = generateLargeDataset(numEntries);
+ const nextList = generateLargeDataset(numEntries);
+
+ nextList[100].value = "updated-value-100"; // 1 updated entry
+ nextList[20_000].value = "updated-value-20000"; // Another updated entry
+ nextList.push({ id: numEntries, value: `new-value-${numEntries}` }); // 1 added entry
+
+ const diffListener = streamListDiffClient<{ id: number; value: string }>(
+ prevList,
+ nextList,
+ "id",
+ {
+ chunksSize: 10_000,
+ },
+ );
+
+ const diffs: StreamListDiff<{ id: number; value: string }>[] = [];
+
+ diffListener.on("data", (chunk) => {
+ diffs.push(...chunk);
+ });
+
+ diffListener.on("finish", () => {
+ try {
+ const updatedEntries = diffs.filter((d) => d.status === "updated");
+ const addedEntries = diffs.filter((d) => d.status === "added");
+ const deletedEntries = diffs.filter((d) => d.status === "deleted");
+ const equalEntries = diffs.filter((d) => d.status === "equal");
+
+ expect(updatedEntries.length).toBe(2);
+ expect(addedEntries.length).toBe(1);
+ expect(deletedEntries.length).toBe(0);
+ expect(equalEntries.length).toBe(99998);
+ done();
+ } catch (err) {
+ done(err);
+ }
+ });
+
+ diffListener.on("error", (err) => done(err));
+ });
+});
diff --git a/src/lib/stream-list-diff/index.ts b/src/lib/stream-list-diff/index.ts
index e949619..9e512c8 100644
--- a/src/lib/stream-list-diff/index.ts
+++ b/src/lib/stream-list-diff/index.ts
@@ -1,327 +1,2 @@
-import {
- DEFAULT_LIST_STREAM_OPTIONS,
- ListStreamOptions,
- ReferenceProperty,
- StreamListDiff,
- StreamReferences,
-} from "@models/stream";
-import { LIST_STATUS } from "@models/list";
-import { isObject } from "@lib/utils";
-import {
- Emitter,
- EmitterEvents,
- EventEmitter,
- StreamListener,
- StreamEvent,
-} from "./emitter";
-
-function outputDiffChunk<T extends Record<string, unknown>>(
- emitter: Emitter<T>,
-) {
- let chunks: StreamListDiff<T>[] = [];
-
- return function handleDiffChunk(
- chunk: StreamListDiff<T>,
- isLastChunk: boolean,
- options: ListStreamOptions,
- ): void {
- const showChunk = options?.showOnly
- ? options?.showOnly.includes(chunk.status)
- : true;
- if (!showChunk) {
- return;
- }
- if ((options.chunksSize as number) > 0) {
- chunks.push(chunk);
- if (chunks.length >= (options.chunksSize as number) || isLastChunk) {
- const output = chunks;
- chunks = [];
- return emitter.emit(StreamEvent.Data, output);
- } else {
- return;
- }
- }
- return emitter.emit(StreamEvent.Data, [chunk]);
- };
-}
-
-function formatSingleListStreamDiff<T extends Record<string, unknown>>(
- list: T[],
- isPrevious: boolean,
- status: LIST_STATUS,
- options: ListStreamOptions,
-): StreamListDiff<T>[] | null {
- let isValid = true;
- const diff: StreamListDiff<T>[] = [];
- for (let i = 0; i < list.length; i++) {
- const data = list[i];
- if (!isObject(data)) {
- isValid = false;
- break;
- }
- diff.push({
- previousValue: isPrevious ? data : null,
- currentValue: isPrevious ? null : data,
- prevIndex: status === LIST_STATUS.ADDED ? null : i,
- newIndex: status === LIST_STATUS.ADDED ? i : null,
- indexDiff: null,
- status,
- });
- }
- if (!isValid) {
- return null;
- }
- if (options.showOnly && options.showOnly.length > 0) {
- return diff.filter((value) => options.showOnly?.includes(value.status));
- }
- return diff;
-}
-
-function isValidChunkSize(
- chunksSize: ListStreamOptions["chunksSize"],
-): boolean {
- if (!chunksSize) return true;
- const sign = String(Math.sign(chunksSize));
- return sign !== "-1" && sign !== "NaN";
-}
-
-function isDataValid<T extends Record<string, unknown>>(
- data: T,
- referenceProperty: ReferenceProperty<T>,
- listType: "prevList" | "nextList",
-): { isValid: boolean; message?: string } {
- if (!isObject(data)) {
- return {
- isValid: false,
- message: `Your ${listType} must only contain valid objects. Found '${data}'`,
- };
- }
- if (!Object.hasOwn(data, referenceProperty)) {
- return {
- isValid: false,
- message: `The reference property '${String(referenceProperty)}' is not available in all the objects of your ${listType}.`,
- };
- }
- return {
- isValid: true,
- message: "",
- };
-}
-
-function getDiffChunks<T extends Record<string, unknown>>(
- prevList: T[] = [],
- nextList: T[] = [],
- referenceProperty: ReferenceProperty<T>,
- emitter: Emitter<T>,
- options: ListStreamOptions = DEFAULT_LIST_STREAM_OPTIONS,
-): void {
- if (!isValidChunkSize(options?.chunksSize)) {
- return emitter.emit(
- StreamEvent.Error,
- new Error(
- `The chunk size can't be negative. You entered the value '${options.chunksSize}'`,
- ),
- );
- }
- if (prevList.length === 0 && nextList.length === 0) {
- return emitter.emit(StreamEvent.Finish);
- }
- const handleDiffChunk = outputDiffChunk<T>(emitter);
- if (prevList.length === 0) {
- const nextDiff = formatSingleListStreamDiff(
- nextList as T[],
- false,
- LIST_STATUS.ADDED,
- options,
- );
- if (!nextDiff) {
- emitter.emit(
- StreamEvent.Error,
- new Error("Your nextList must only contain valid objects."),
- );
- return emitter.emit(StreamEvent.Finish);
- }
- nextDiff?.forEach((data, i) =>
- handleDiffChunk(data, i === nextDiff.length - 1, options),
- );
- return emitter.emit(StreamEvent.Finish);
- }
- if (nextList.length === 0) {
- const prevDiff = formatSingleListStreamDiff(
- prevList as T[],
- true,
- LIST_STATUS.DELETED,
- options,
- );
- if (!prevDiff) {
- emitter.emit(
- StreamEvent.Error,
- new Error("Your prevList must only contain valid objects."),
- );
- return emitter.emit(StreamEvent.Finish);
- }
- prevDiff?.forEach((data, i) =>
- handleDiffChunk(data, i === prevDiff.length - 1, options),
- );
- return emitter.emit(StreamEvent.Finish);
- }
- const listsReferences: StreamReferences<T> = new Map();
- for (let i = 0; i < prevList.length; i++) {
- const data = prevList[i];
- if (data) {
- const { isValid, message } = isDataValid(
- data,
- referenceProperty,
- "prevList",
- );
- if (!isValid) {
- emitter.emit(StreamEvent.Error, new Error(message));
- emitter.emit(StreamEvent.Finish);
- break;
- }
- listsReferences.set(String(data[referenceProperty]), {
- prevIndex: i,
- nextIndex: undefined,
- });
- }
- }
-
- const totalChunks = listsReferences.size;
-
- for (let i = 0; i < nextList.length; i++) {
- const data = nextList[i];
- if (data) {
- const { isValid, message } = isDataValid(
- data,
- referenceProperty,
- "nextList",
- );
- if (!isValid) {
- emitter.emit(StreamEvent.Error, new Error(message));
- emitter.emit(StreamEvent.Finish);
- break;
- }
- const listReference = listsReferences.get(
- String(data[referenceProperty]),
- );
- if (listReference) {
- listReference.nextIndex = i;
- } else {
- handleDiffChunk(
- {
- previousValue: null,
- currentValue: data,
- prevIndex: null,
- newIndex: i,
- indexDiff: null,
- status: LIST_STATUS.ADDED,
- },
- totalChunks > 0 ? false : i === nextList.length - 1,
- options,
- );
- }
- }
- }
-
- let streamedChunks = 0;
-
- for (const [key, data] of listsReferences.entries()) {
- streamedChunks++;
- const isLastChunk = totalChunks === streamedChunks;
-
- if (typeof data.nextIndex === "undefined") {
- handleDiffChunk(
- {
- previousValue: prevList[data.prevIndex],
- currentValue: null,
- prevIndex: data.prevIndex,
- newIndex: null,
- indexDiff: null,
- status: LIST_STATUS.DELETED,
- },
- isLastChunk,
- options,
- );
- } else {
- const prevData = prevList[data.prevIndex];
- const nextData = nextList[data.nextIndex];
- const isDataEqual = JSON.stringify(prevData) === JSON.stringify(nextData);
- const indexDiff = data.nextIndex - data.prevIndex;
- if (isDataEqual) {
- if (indexDiff === 0) {
- handleDiffChunk(
- {
- previousValue: prevList[data.prevIndex],
- currentValue: nextList[data.nextIndex],
- prevIndex: data.prevIndex,
- newIndex: data.nextIndex,
- indexDiff: 0,
- status: LIST_STATUS.EQUAL,
- },
- isLastChunk,
- options,
- );
- } else {
- handleDiffChunk(
- {
- previousValue: prevList[data.prevIndex],
- currentValue: nextList[data.nextIndex],
- prevIndex: data.prevIndex,
- newIndex: data.nextIndex,
- indexDiff,
- status: options.considerMoveAsUpdate
- ? LIST_STATUS.UPDATED
- : LIST_STATUS.MOVED,
- },
- isLastChunk,
- options,
- );
- }
- } else {
- handleDiffChunk(
- {
- previousValue: prevList[data.prevIndex],
- currentValue: nextList[data.nextIndex],
- prevIndex: data.prevIndex,
- newIndex: data.nextIndex,
- indexDiff,
- status: LIST_STATUS.UPDATED,
- },
- isLastChunk,
- options,
- );
- }
- }
- listsReferences.delete(key);
- }
-
- return emitter.emit(StreamEvent.Finish);
-}
-
-/**
- * Streams the diff of two object lists
- * @param {Record<string, unknown>[]} prevList - The original object list.
- * @param {Record<string, unknown>[]} nextList - The new object list.
- * @param {ReferenceProperty<T>} referenceProperty - A common property in all the objects of your lists (e.g. `id`)
- * @param {ListStreamOptions} options - Options to refine your output.
- - `chunksSize`: the number of object diffs returned by each streamed chunk. (e.g. `0` = 1 object diff by chunk, `10` = 10 object diffs by chunk).
- - `showOnly`: returns only the values whose status you are interested in. (e.g. `["added", "equal"]`)
- - `considerMoveAsUpdate`: if set to `true` a `moved` object will be considered as `updated`
- * @returns EventEmitter
- */
-export function streamListDiff<T extends Record<string, unknown>>(
- prevList: T[],
- nextList: T[],
- referenceProperty: ReferenceProperty<T>,
- options: ListStreamOptions = DEFAULT_LIST_STREAM_OPTIONS,
-): StreamListener<T> {
- const emitter = new EventEmitter<EmitterEvents<T>>();
- setTimeout(() => {
- try {
- getDiffChunks(prevList, nextList, referenceProperty, emitter, options);
- } catch (err) {
- return emitter.emit(StreamEvent.Error, err as Error);
- }
- }, 0);
- return emitter as StreamListener<T>;
-}
+export { streamListDiffClient } from "./client";
+export { streamListDiff } from "./server";
diff --git a/src/lib/stream-list-diff/server/index.ts b/src/lib/stream-list-diff/server/index.ts
new file mode 100644
index 0000000..dc327e7
--- /dev/null
+++ b/src/lib/stream-list-diff/server/index.ts
@@ -0,0 +1,261 @@
+import { createReadStream } from "fs";
+import { Readable, Transform } from "stream";
+import { LIST_STATUS } from "@models/list";
+import {
+ DataBuffer,
+ DEFAULT_LIST_STREAM_OPTIONS,
+ FilePath,
+ ListStreamOptions,
+ ReferenceProperty,
+} from "@models/stream";
+import { isClient } from "@lib/utils";
+import {
+ Emitter,
+ EmitterEvents,
+ EventEmitter,
+ StreamListener,
+ StreamEvent,
+} from "../emitter";
+import { isDataValid, isValidChunkSize, outputDiffChunk } from "../utils";
+
+async function getDiffChunks<T extends Record<string, unknown>>(
+ prevStream: Readable,
+ nextStream: Readable,
+ referenceProperty: ReferenceProperty<T>,
+ emitter: Emitter<T>,
+ options: ListStreamOptions = DEFAULT_LIST_STREAM_OPTIONS,
+): Promise<void> {
+ if (!isValidChunkSize(options?.chunksSize)) {
+ return emitter.emit(
+ StreamEvent.Error,
+ new Error(
+ `The chunk size can't be negative. You entered the value '${options.chunksSize}'`,
+ ),
+ );
+ }
+ const { handleDiffChunk, releaseLastChunks } = outputDiffChunk<T>(emitter);
+ const prevDataBuffer: DataBuffer<T> = new Map();
+ const nextDataBuffer: DataBuffer<T> = new Map();
+ let currentPrevIndex = 0;
+ let currentNextIndex = 0;
+
+ async function processPrevStreamChunk(chunk: T) {
+ const { isValid, message } = isDataValid(
+ chunk,
+ referenceProperty,
+ "prevList",
+ );
+ if (!isValid) {
+ emitter.emit(StreamEvent.Error, new Error(message));
+ emitter.emit(StreamEvent.Finish);
+ return;
+ }
+ const ref = chunk[referenceProperty] as ReferenceProperty<T>;
+ const relatedChunk = nextDataBuffer.get(ref);
+
+ if (relatedChunk) {
+ nextDataBuffer.delete(ref);
+ const isDataEqual =
+ JSON.stringify(chunk) === JSON.stringify(relatedChunk.data);
+ const indexDiff = (relatedChunk.index as number) - currentPrevIndex;
+ if (isDataEqual) {
+ handleDiffChunk(
+ {
+ previousValue: chunk,
+ currentValue: relatedChunk.data,
+ prevIndex: currentPrevIndex,
+ newIndex: relatedChunk.index,
+ indexDiff,
+ status:
+ indexDiff === 0
+ ? LIST_STATUS.EQUAL
+ : options.considerMoveAsUpdate
+ ? LIST_STATUS.UPDATED
+ : LIST_STATUS.MOVED,
+ },
+ options,
+ );
+ } else {
+ handleDiffChunk(
+ {
+ previousValue: chunk,
+ currentValue: relatedChunk.data,
+ prevIndex: currentPrevIndex,
+ newIndex: relatedChunk.index,
+ indexDiff,
+ status: LIST_STATUS.UPDATED,
+ },
+ options,
+ );
+ }
+ } else {
+ prevDataBuffer.set(ref, { data: chunk, index: currentPrevIndex });
+ }
+ currentPrevIndex++;
+ }
+
+ async function processNextStreamChunk(chunk: T) {
+ const { isValid, message } = isDataValid(
+ chunk,
+ referenceProperty,
+ "nextList",
+ );
+ if (!isValid) {
+ emitter.emit(StreamEvent.Error, new Error(message));
+ emitter.emit(StreamEvent.Finish);
+ return;
+ }
+ const ref = chunk[referenceProperty] as ReferenceProperty<T>;
+ const relatedChunk = prevDataBuffer.get(ref);
+
+ if (relatedChunk) {
+ prevDataBuffer.delete(ref);
+ const isDataEqual =
+ JSON.stringify(chunk) === JSON.stringify(relatedChunk.data);
+ const indexDiff = currentNextIndex - (relatedChunk.index as number);
+ if (isDataEqual) {
+ handleDiffChunk(
+ {
+ previousValue: relatedChunk.data,
+ currentValue: chunk,
+ prevIndex: relatedChunk.index,
+ newIndex: currentNextIndex,
+ indexDiff,
+ status:
+ indexDiff === 0
+ ? LIST_STATUS.EQUAL
+ : options.considerMoveAsUpdate
+ ? LIST_STATUS.UPDATED
+ : LIST_STATUS.MOVED,
+ },
+ options,
+ );
+ } else {
+ handleDiffChunk(
+ {
+ previousValue: relatedChunk.data,
+ currentValue: chunk,
+ prevIndex: relatedChunk.index,
+ newIndex: currentNextIndex,
+ indexDiff,
+ status: LIST_STATUS.UPDATED,
+ },
+ options,
+ );
+ }
+ } else {
+ nextDataBuffer.set(ref, { data: chunk, index: currentNextIndex });
+ }
+ currentNextIndex++;
+ }
+
+ const prevStreamReader = async () => {
+ for await (const chunk of prevStream) {
+ await processPrevStreamChunk(chunk);
+ }
+ };
+
+ const nextStreamReader = async () => {
+ for await (const chunk of nextStream) {
+ await processNextStreamChunk(chunk);
+ }
+ };
+ await Promise.all([prevStreamReader(), nextStreamReader()]);
+
+ for (const [key, chunk] of prevDataBuffer.entries()) {
+ handleDiffChunk(
+ {
+ previousValue: chunk.data,
+ currentValue: null,
+ prevIndex: chunk.index,
+ newIndex: null,
+ indexDiff: null,
+ status: LIST_STATUS.DELETED,
+ },
+ options,
+ );
+ prevDataBuffer.delete(key);
+ }
+ for (const [key, chunk] of nextDataBuffer.entries()) {
+ handleDiffChunk(
+ {
+ previousValue: null,
+ currentValue: chunk.data,
+ prevIndex: null,
+ newIndex: chunk.index,
+ indexDiff: null,
+ status: LIST_STATUS.ADDED,
+ },
+ options,
+ );
+ nextDataBuffer.delete(key);
+ }
+ releaseLastChunks();
+ return emitter.emit(StreamEvent.Finish);
+}
+
+function getValidStream<T>(
+ input: Readable | FilePath | T[],
+ listType: "prevList" | "nextList",
+): Readable {
+ if (input instanceof Readable) {
+ return input;
+ }
+
+ if (Array.isArray(input)) {
+ return Readable.from(input, { objectMode: true });
+ }
+
+ if (typeof input === "string") {
+ return createReadStream(input, { encoding: "utf8" }).pipe(
+ new Transform({
+ objectMode: true,
+ transform(chunk, _, callback) {
+ try {
+ const data: T = JSON.parse(chunk.toString());
+ if (Array.isArray(data)) {
+ for (let i = 0; i < data.length; i++) {
+ this.push(data[i]);
+ }
+ } else {
+ this.push(data);
+ }
+ callback();
+ } catch (err) {
+ callback(err as Error);
+ }
+ },
+ }),
+ );
+ }
+
+ throw new Error(`Invalid ${listType}. Expected Readable, Array, or File.`);
+}
+
+export function streamListDiff<T extends Record<string, unknown>>(
+ prevStream: Readable | FilePath | T[],
+ nextStream: Readable | FilePath | T[],
+ referenceProperty: ReferenceProperty<T>,
+ options: ListStreamOptions = DEFAULT_LIST_STREAM_OPTIONS,
+): StreamListener<T> {
+ if (isClient()) {
+ throw new Error(
+ "streamListDiff can only be used in Node environment. Please use streamListDiffClient instead.",
+ );
+ }
+ const emitter = new EventEmitter<EmitterEvents<T>>();
+ setTimeout(async () => {
+ try {
+ await getDiffChunks(
+ getValidStream(prevStream, "prevList"),
+ getValidStream(nextStream, "nextList"),
+ referenceProperty,
+ emitter,
+ options,
+ );
+ } catch (err) {
+ return emitter.emit(StreamEvent.Error, err as Error);
+ }
+ }, 0);
+ return emitter as StreamListener<T>;
+}
diff --git a/src/lib/stream-list-diff/stream-list-diff.test.ts b/src/lib/stream-list-diff/server/stream-list-diff.test.ts
similarity index 74%
rename from src/lib/stream-list-diff/stream-list-diff.test.ts
rename to src/lib/stream-list-diff/server/stream-list-diff.test.ts
index 5d8795c..f1751d0 100644
--- a/src/lib/stream-list-diff/stream-list-diff.test.ts
+++ b/src/lib/stream-list-diff/server/stream-list-diff.test.ts
@@ -1,8 +1,10 @@
+import path from "path";
+import { Readable } from "stream";
import { LIST_STATUS } from "@models/list";
-import { streamListDiff } from ".";
import { StreamListDiff } from "@models/stream";
+import { streamListDiff } from ".";
-describe("streamListDiff data", () => {
+describe("data emission", () => {
it("emits 'data' event and consider the all the nextList added if no prevList is provided", (done) => {
const nextList = [
{ id: 1, name: "Item 1" },
@@ -68,6 +70,7 @@ describe("streamListDiff data", () => {
expect(chunk).toStrictEqual(expectedChunks);
chunkCount++;
});
+ diff.on("error", (err) => console.error("shiiiite", err));
diff.on("finish", () => {
expect(chunkCount).toBe(1);
done();
@@ -87,12 +90,12 @@ describe("streamListDiff data", () => {
const expectedChunks = [
[
{
- previousValue: null,
- currentValue: { id: 3, name: "Item 3" },
- prevIndex: null,
- newIndex: 1,
- indexDiff: null,
- status: LIST_STATUS.ADDED,
+ previousValue: { id: 2, name: "Item 2" },
+ currentValue: { id: 2, name: "Item 2" },
+ prevIndex: 1,
+ newIndex: 0,
+ indexDiff: -1,
+ status: LIST_STATUS.MOVED,
},
],
[
@@ -107,12 +110,12 @@ describe("streamListDiff data", () => {
],
[
{
- previousValue: { id: 2, name: "Item 2" },
- currentValue: { id: 2, name: "Item 2" },
- prevIndex: 1,
- newIndex: 0,
- indexDiff: -1,
- status: LIST_STATUS.MOVED,
+ previousValue: null,
+ currentValue: { id: 3, name: "Item 3" },
+ prevIndex: null,
+ newIndex: 1,
+ indexDiff: null,
+ status: LIST_STATUS.ADDED,
},
],
];
@@ -153,18 +156,12 @@ describe("streamListDiff data", () => {
{ id: 9, name: "Item 9" },
{ id: 8, name: "Item 8" },
];
- const diff = streamListDiff(prevList, nextList, "id", { chunksSize: 5 });
+ const diff = streamListDiff(prevList, nextList, "id", {
+ chunksSize: 5,
+ });
const expectedChunks = [
[
- {
- previousValue: null,
- currentValue: { id: 11, name: "Item 11" },
- prevIndex: null,
- newIndex: 7,
- indexDiff: null,
- status: LIST_STATUS.ADDED,
- },
{
previousValue: { id: 1, name: "Item 1" },
currentValue: { id: 1, name: "Item 1" },
@@ -189,16 +186,6 @@ describe("streamListDiff data", () => {
indexDiff: 0,
status: LIST_STATUS.EQUAL,
},
- {
- previousValue: { id: 4, name: "Item 4" },
- currentValue: null,
- prevIndex: 3,
- newIndex: null,
- indexDiff: null,
- status: LIST_STATUS.DELETED,
- },
- ],
- [
{
previousValue: { id: 5, name: "Item 5" },
currentValue: { id: 5, name: "Item 5" },
@@ -215,6 +202,8 @@ describe("streamListDiff data", () => {
indexDiff: -1,
status: LIST_STATUS.UPDATED,
},
+ ],
+ [
{
previousValue: { id: 7, name: "Item 7" },
currentValue: { id: 7, name: "Item 7" },
@@ -223,14 +212,6 @@ describe("streamListDiff data", () => {
indexDiff: -1,
status: LIST_STATUS.MOVED,
},
- {
- previousValue: { id: 8, name: "Item 8" },
- currentValue: { id: 8, name: "Item 8" },
- prevIndex: 7,
- newIndex: 9,
- indexDiff: 2,
- status: LIST_STATUS.MOVED,
- },
{
previousValue: { id: 9, name: "Item 9" },
currentValue: { id: 9, name: "Item 9" },
@@ -239,8 +220,6 @@ describe("streamListDiff data", () => {
indexDiff: 0,
status: LIST_STATUS.EQUAL,
},
- ],
- [
{
previousValue: { id: 10, name: "Item 10" },
currentValue: { id: 10, name: "Item 10" },
@@ -249,6 +228,32 @@ describe("streamListDiff data", () => {
indexDiff: -3,
status: LIST_STATUS.MOVED,
},
+ {
+ previousValue: { id: 8, name: "Item 8" },
+ currentValue: { id: 8, name: "Item 8" },
+ prevIndex: 7,
+ newIndex: 9,
+ indexDiff: 2,
+ status: LIST_STATUS.MOVED,
+ },
+ {
+ previousValue: { id: 4, name: "Item 4" },
+ currentValue: null,
+ prevIndex: 3,
+ newIndex: null,
+ indexDiff: null,
+ status: LIST_STATUS.DELETED,
+ },
+ ],
+ [
+ {
+ previousValue: null,
+ currentValue: { id: 11, name: "Item 11" },
+ prevIndex: null,
+ newIndex: 7,
+ indexDiff: null,
+ status: LIST_STATUS.ADDED,
+ },
],
];
@@ -271,23 +276,19 @@ describe("streamListDiff data", () => {
{ id: 3, name: "Item 3" },
{ id: 4, name: "Item 4" },
];
+
const nextList = [
{ id: 1, name: "Item 1" },
{ id: 2, name: "Item Two" },
{ id: 3, name: "Item 3" },
{ id: 5, name: "Item 5" },
];
- const diff = streamListDiff(prevList, nextList, "id", { chunksSize: 150 });
+
+ const diff = streamListDiff(prevList, nextList, "id", {
+ chunksSize: 5,
+ });
const expectedChunks = [
- {
- previousValue: null,
- currentValue: { id: 5, name: "Item 5" },
- prevIndex: null,
- newIndex: 3,
- indexDiff: null,
- status: LIST_STATUS.ADDED,
- },
{
previousValue: { id: 1, name: "Item 1" },
currentValue: { id: 1, name: "Item 1" },
@@ -320,6 +321,14 @@ describe("streamListDiff data", () => {
indexDiff: null,
status: LIST_STATUS.DELETED,
},
+ {
+ previousValue: null,
+ currentValue: { id: 5, name: "Item 5" },
+ prevIndex: null,
+ newIndex: 3,
+ indexDiff: null,
+ status: LIST_STATUS.ADDED,
+ },
];
let chunkCount = 0;
@@ -327,7 +336,7 @@ describe("streamListDiff data", () => {
expect(chunk).toStrictEqual(expectedChunks);
chunkCount++;
});
-
+ diff.on("error", (err) => console.error("sheeeet", err));
diff.on("finish", () => {
expect(chunkCount).toBe(1);
done();
@@ -353,12 +362,12 @@ describe("streamListDiff data", () => {
const expectedChunks = [
{
- previousValue: null,
- currentValue: { id: 5, name: "Item 5" },
- prevIndex: null,
- newIndex: 3,
- indexDiff: null,
- status: LIST_STATUS.ADDED,
+ previousValue: { id: 2, name: "Item 2" },
+ currentValue: { id: 2, name: "Item Two" },
+ prevIndex: 1,
+ newIndex: 0,
+ indexDiff: -1,
+ status: LIST_STATUS.UPDATED,
},
{
previousValue: { id: 1, name: "Item 1" },
@@ -368,14 +377,6 @@ describe("streamListDiff data", () => {
indexDiff: 1,
status: LIST_STATUS.UPDATED,
},
- {
- previousValue: { id: 2, name: "Item 2" },
- currentValue: { id: 2, name: "Item Two" },
- prevIndex: 1,
- newIndex: 0,
- indexDiff: -1,
- status: LIST_STATUS.UPDATED,
- },
{
previousValue: { id: 3, name: "Item 3" },
currentValue: { id: 3, name: "Item 3" },
@@ -392,6 +393,14 @@ describe("streamListDiff data", () => {
indexDiff: null,
status: LIST_STATUS.DELETED,
},
+ {
+ previousValue: null,
+ currentValue: { id: 5, name: "Item 5" },
+ prevIndex: null,
+ newIndex: 3,
+ indexDiff: null,
+ status: LIST_STATUS.ADDED,
+ },
];
let chunkCount = 0;
@@ -424,14 +433,6 @@ describe("streamListDiff data", () => {
});
const expectedChunks = [
- {
- previousValue: null,
- currentValue: { id: 5, name: "Item 5" },
- prevIndex: null,
- newIndex: 3,
- indexDiff: null,
- status: LIST_STATUS.ADDED,
- },
{
previousValue: { id: 4, name: "Item 4" },
currentValue: null,
@@ -440,6 +441,14 @@ describe("streamListDiff data", () => {
indexDiff: null,
status: LIST_STATUS.DELETED,
},
+ {
+ previousValue: null,
+ currentValue: { id: 5, name: "Item 5" },
+ prevIndex: null,
+ newIndex: 3,
+ indexDiff: null,
+ status: LIST_STATUS.ADDED,
+ },
];
let chunkCount = 0;
@@ -506,18 +515,12 @@ describe("streamListDiff data", () => {
{ id: 9, name: "Item 9" },
{ id: 8, name: "Item 8" },
];
- const diff = streamListDiff(prevList, nextList, "id", { chunksSize: 5 });
+ const diff = streamListDiff(prevList, nextList, "id", {
+ chunksSize: 5,
+ });
const expectedChunks = [
[
- {
- previousValue: null,
- currentValue: { id: 11, name: "Item 11" },
- prevIndex: null,
- newIndex: 7,
- indexDiff: null,
- status: LIST_STATUS.ADDED,
- },
{
previousValue: {
id: 1,
@@ -558,20 +561,6 @@ describe("streamListDiff data", () => {
indexDiff: 0,
status: LIST_STATUS.EQUAL,
},
- {
- previousValue: {
- id: 4,
- name: "Item 4",
- user: { role: "reader", hobbies: ["video games", "fishing"] },
- },
- currentValue: null,
- prevIndex: 3,
- newIndex: null,
- indexDiff: null,
- status: LIST_STATUS.DELETED,
- },
- ],
- [
{
previousValue: { id: 5, name: "Item 5" },
currentValue: { id: 5, name: "Item 5" },
@@ -596,6 +585,8 @@ describe("streamListDiff data", () => {
indexDiff: -1,
status: LIST_STATUS.UPDATED,
},
+ ],
+ [
{
previousValue: { id: 7, name: "Item 7" },
currentValue: { id: 7, name: "Item 7" },
@@ -604,14 +595,6 @@ describe("streamListDiff data", () => {
indexDiff: -1,
status: LIST_STATUS.MOVED,
},
- {
- previousValue: { id: 8, name: "Item 8" },
- currentValue: { id: 8, name: "Item 8" },
- prevIndex: 7,
- newIndex: 9,
- indexDiff: 2,
- status: LIST_STATUS.MOVED,
- },
{
previousValue: { id: 9, name: "Item 9" },
currentValue: { id: 9, name: "Item 9" },
@@ -620,8 +603,6 @@ describe("streamListDiff data", () => {
indexDiff: 0,
status: LIST_STATUS.EQUAL,
},
- ],
- [
{
previousValue: {
id: 10,
@@ -646,6 +627,36 @@ describe("streamListDiff data", () => {
indexDiff: -3,
status: LIST_STATUS.MOVED,
},
+ {
+ previousValue: { id: 8, name: "Item 8" },
+ currentValue: { id: 8, name: "Item 8" },
+ prevIndex: 7,
+ newIndex: 9,
+ indexDiff: 2,
+ status: LIST_STATUS.MOVED,
+ },
+ {
+ previousValue: {
+ id: 4,
+ name: "Item 4",
+ user: { role: "reader", hobbies: ["video games", "fishing"] },
+ },
+ currentValue: null,
+ prevIndex: 3,
+ newIndex: null,
+ indexDiff: null,
+ status: LIST_STATUS.DELETED,
+ },
+ ],
+ [
+ {
+ previousValue: null,
+ currentValue: { id: 11, name: "Item 11" },
+ prevIndex: null,
+ newIndex: 7,
+ indexDiff: null,
+ status: LIST_STATUS.ADDED,
+ },
],
];
@@ -663,7 +674,158 @@ describe("streamListDiff data", () => {
});
});
-describe("streamListDiff finish", () => {
+describe("input handling", () => {
+ const prevList = [
+ { id: 1, name: "Item 1" },
+ { id: 2, name: "Item 2" },
+ { id: 3, name: "Item 3" },
+ { id: 4, name: "Item 4" },
+ ];
+ const nextList = [
+ { id: 1, name: "Item 1" },
+ { id: 2, name: "Item Two" },
+ { id: 3, name: "Item 3" },
+ { id: 5, name: "Item 5" },
+ ];
+ const expectedChunks = [
+ {
+ previousValue: { id: 1, name: "Item 1" },
+ currentValue: { id: 1, name: "Item 1" },
+ prevIndex: 0,
+ newIndex: 0,
+ indexDiff: 0,
+ status: LIST_STATUS.EQUAL,
+ },
+ {
+ previousValue: { id: 2, name: "Item 2" },
+ currentValue: { id: 2, name: "Item Two" },
+ prevIndex: 1,
+ newIndex: 1,
+ indexDiff: 0,
+ status: LIST_STATUS.UPDATED,
+ },
+ {
+ previousValue: { id: 3, name: "Item 3" },
+ currentValue: { id: 3, name: "Item 3" },
+ prevIndex: 2,
+ newIndex: 2,
+ indexDiff: 0,
+ status: LIST_STATUS.EQUAL,
+ },
+ {
+ previousValue: { id: 4, name: "Item 4" },
+ currentValue: null,
+ prevIndex: 3,
+ newIndex: null,
+ indexDiff: null,
+ status: LIST_STATUS.DELETED,
+ },
+ {
+ previousValue: null,
+ currentValue: { id: 5, name: "Item 5" },
+ prevIndex: null,
+ newIndex: 3,
+ indexDiff: null,
+ status: LIST_STATUS.ADDED,
+ },
+ ];
+
+ it("handles two readable streams", (done) => {
+ const prevStream = Readable.from(prevList, { objectMode: true });
+ const nextStream = Readable.from(nextList, { objectMode: true });
+
+ const diff = streamListDiff(prevStream, nextStream, "id", {
+ chunksSize: 5,
+ });
+
+ let chunkCount = 0;
+ diff.on("data", (chunk) => {
+ expect(chunk).toStrictEqual(expectedChunks);
+ chunkCount++;
+ });
+ diff.on("error", (err) => console.error("sheeeet", err));
+ diff.on("finish", () => {
+ expect(chunkCount).toBe(1);
+ done();
+ });
+ });
+ it("handles two local files", (done) => {
+ const prevFile = path.resolve(__dirname, "../../../mocks/prevList.json");
+ const nextFile = path.resolve(__dirname, "../../../mocks/nextList.json");
+
+ const diff = streamListDiff(prevFile, nextFile, "id", {
+ chunksSize: 5,
+ });
+
+ let chunkCount = 0;
+ diff.on("data", (chunk) => {
+ expect(chunk).toStrictEqual(expectedChunks);
+ chunkCount++;
+ });
+ diff.on("error", (err) => console.error("sheeeet", err));
+ diff.on("finish", () => {
+ expect(chunkCount).toBe(1);
+ done();
+ });
+ });
+ it("handles a readable stream against a local file", (done) => {
+ const prevStream = Readable.from(prevList, { objectMode: true });
+ const nextFile = path.resolve(__dirname, "../../../mocks/nextList.json");
+
+ const diff = streamListDiff(prevStream, nextFile, "id", {
+ chunksSize: 5,
+ });
+
+ let chunkCount = 0;
+ diff.on("data", (chunk) => {
+ expect(chunk).toStrictEqual(expectedChunks);
+ chunkCount++;
+ });
+ diff.on("error", (err) => console.error("sheeeet", err));
+ diff.on("finish", () => {
+ expect(chunkCount).toBe(1);
+ done();
+ });
+ });
+ it("handles a readable stream against an array", (done) => {
+ const prevStream = Readable.from(prevList, { objectMode: true });
+
+ const diff = streamListDiff(prevStream, nextList, "id", {
+ chunksSize: 5,
+ });
+
+ let chunkCount = 0;
+ diff.on("data", (chunk) => {
+ expect(chunk).toStrictEqual(expectedChunks);
+ chunkCount++;
+ });
+ diff.on("error", (err) => console.error("sheeeet", err));
+ diff.on("finish", () => {
+ expect(chunkCount).toBe(1);
+ done();
+ });
+ });
+ it("handles a local file against an array", (done) => {
+ const prevFile = path.resolve(__dirname, "../../../mocks/prevList.json");
+
+ const diff = streamListDiff(prevFile, nextList, "id", {
+ chunksSize: 5,
+ });
+
+ let chunkCount = 0;
+ diff.on("data", (chunk) => {
+ expect(chunk).toStrictEqual(expectedChunks);
+ chunkCount++;
+ });
+ diff.on("error", (err) => console.error("sheeeet", err));
+ diff.on("finish", () => {
+ expect(chunkCount).toBe(1);
+ done();
+ });
+ });
+});
+
+describe("finish event", () => {
it("emits 'finish' event if no prevList nor nextList is provided", (done) => {
const diff = streamListDiff([], [], "id");
diff.on("finish", () => done());
@@ -682,7 +844,7 @@ describe("streamListDiff finish", () => {
});
});
-describe("streamListDiff error", () => {
+describe("error event", () => {
test("emits 'error' event when prevList has invalid data", (done) => {
const prevList = [
{ id: 1, name: "Item 1" },
@@ -768,7 +930,9 @@ describe("streamListDiff error", () => {
];
const nextList = [{ id: 1, name: "Item 1" }, { name: "Item 2" }];
- const diff = streamListDiff(prevList, nextList, "id", { chunksSize: -3 });
+ const diff = streamListDiff(prevList, nextList, "id", {
+ chunksSize: -3,
+ });
diff.on("error", (err) => {
expect(err["message"]).toEqual(
@@ -777,91 +941,86 @@ describe("streamListDiff error", () => {
done();
});
});
-});
-describe("Performance", () => {
- it("should correctly stream diff for 10.000 entries", (done) => {
- const generateLargeList = (size: number, idPrefix: string) => {
- return Array.from({ length: size }, (_, i) => ({
- id: `${idPrefix}-${i}`,
- value: i,
- }));
- };
- const prevList = generateLargeList(10_000, "prev");
- const nextList = [
- ...generateLargeList(5000, "prev"),
- ...generateLargeList(5000, "next"),
- ];
+ test("emits 'error' event when the prevList is not a valid type", (done) => {
+ const nextList = [{ id: 1, name: "Item 1" }, { name: "Item 2" }];
- const receivedChunks: StreamListDiff<{ id: string; value: number }>[] = [];
- let chunkCount = 0;
- const diffStream = streamListDiff(prevList, nextList, "id", {
- chunksSize: 1000,
- });
+ // @ts-expect-error - prevList is invalid by design for the test
+ const diff = streamListDiff({ name: "hello" }, nextList, "id");
- diffStream.on("data", (chunk) => {
- receivedChunks.push(...chunk);
- chunkCount++;
+ diff.on("error", (err) => {
+ expect(err["message"]).toEqual(
+ "Invalid prevList. Expected Readable, Array, or File.",
+ );
+ done();
});
+ });
+ test("emits 'error' event when the nextList is not a valid type", (done) => {
+ const prevList = [{ id: 1, name: "Item 1" }, { name: "Item 2" }];
- diffStream.on("finish", () => {
- const deletions = receivedChunks.filter(
- (diff) => diff.status === LIST_STATUS.DELETED,
- );
- const additions = receivedChunks.filter(
- (diff) => diff.status === LIST_STATUS.ADDED,
- );
- const updates = receivedChunks.filter(
- (diff) => diff.status === LIST_STATUS.EQUAL,
+ // @ts-expect-error - nextList is invalid by design for the test
+ const diff = streamListDiff(prevList, null, "id");
+
+ diff.on("error", (err) => {
+ expect(err["message"]).toEqual(
+ "Invalid nextList. Expected Readable, Array, or File.",
);
- expect(receivedChunks.length).toBe(15_000); // 5000 deletions + 5000 equal + 5000 additions
- expect(chunkCount).toBe(15);
- expect(deletions.length).toBe(5000);
- expect(additions.length).toBe(5000);
- expect(updates.length).toBe(5000);
done();
});
});
- it("should correctly stream diff for 100.000 entries", (done) => {
- const generateLargeList = (size: number, idPrefix: string) => {
- return Array.from({ length: size }, (_, i) => ({
- id: `${idPrefix}-${i}`,
- value: i,
- }));
- };
- const prevList = generateLargeList(100_000, "prev");
- const nextList = [
- ...generateLargeList(50000, "prev"),
- ...generateLargeList(50000, "next"),
- ];
+});
- const receivedChunks: StreamListDiff<{ id: string; value: number }>[] = [];
- let chunkCount = 0;
- const diffStream = streamListDiff(prevList, nextList, "id", {
- chunksSize: 10_000,
- });
+const generateLargeDataset = (count: number) => {
+ const data: Array<{ id: number; value: string }> = [];
+ for (let i = 0; i < count; i++) {
+ data.push({ id: i, value: `value-${i}` });
+ }
+ return data;
+};
- diffStream.on("data", (chunk) => {
- receivedChunks.push(...chunk);
- chunkCount++;
+describe("performance", () => {
+ it("process 100.000 in each stream", (done) => {
+ const numEntries = 100_000;
+
+ const prevList = generateLargeDataset(numEntries);
+ const nextList = generateLargeDataset(numEntries);
+
+ nextList[100].value = "updated-value-100"; // 1 updated entry
+ nextList[20_000].value = "updated-value-20000"; // Another updated entry
+ nextList.push({ id: numEntries, value: `new-value-${numEntries}` }); // 1 added entry
+
+ const diffListener = streamListDiff<{ id: number; value: string }>(
+ prevList,
+ nextList,
+ "id",
+ {
+ chunksSize: 10_000,
+ },
+ );
+
+ const diffs: StreamListDiff<{ id: number; value: string }>[] = [];
+
+ diffListener.on("data", (chunk) => {
+ diffs.push(...chunk);
});
- diffStream.on("finish", () => {
- const deletions = receivedChunks.filter(
- (diff) => diff.status === LIST_STATUS.DELETED,
- );
- const additions = receivedChunks.filter(
- (diff) => diff.status === LIST_STATUS.ADDED,
- );
- const updates = receivedChunks.filter(
- (diff) => diff.status === LIST_STATUS.EQUAL,
- );
- expect(receivedChunks.length).toBe(150_000); // 50.000 deletions + 50.000 equal + 50.000 additions
- expect(chunkCount).toBe(15);
- expect(deletions.length).toBe(50000);
- expect(additions.length).toBe(50000);
- expect(updates.length).toBe(50000);
- done();
+ diffListener.on("finish", () => {
+ try {
+ const updatedEntries = diffs.filter((d) => d.status === "updated");
+ const addedEntries = diffs.filter((d) => d.status === "added");
+ const deletedEntries = diffs.filter((d) => d.status === "deleted");
+ const equalEntries = diffs.filter((d) => d.status === "equal");
+
+ expect(updatedEntries.length).toBe(2);
+ expect(addedEntries.length).toBe(1);
+ expect(deletedEntries.length).toBe(0);
+ expect(equalEntries.length).toBe(99998);
+ done();
+ } catch (err) {
+ done(err);
+ }
});
+
+ diffListener.on("error", (err) => done(err));
});
});
diff --git a/src/lib/stream-list-diff/utils.ts b/src/lib/stream-list-diff/utils.ts
new file mode 100644
index 0000000..8356343
--- /dev/null
+++ b/src/lib/stream-list-diff/utils.ts
@@ -0,0 +1,80 @@
+import { isObject } from "@lib/utils";
+import {
+ ListStreamOptions,
+ ReferenceProperty,
+ StreamListDiff,
+} from "@models/stream";
+import { Emitter, StreamEvent } from "./emitter";
+
+export function isValidChunkSize(
+ chunksSize: ListStreamOptions["chunksSize"],
+): boolean {
+ if (!chunksSize) return true;
+ const sign = String(Math.sign(chunksSize));
+ return sign !== "-1" && sign !== "NaN";
+}
+
+export function isDataValid<T extends Record<string, unknown>>(
+ data: T,
+ referenceProperty: ReferenceProperty<T>,
+ listType: "prevList" | "nextList",
+): { isValid: boolean; message?: string } {
+ if (!isObject(data)) {
+ return {
+ isValid: false,
+ message: `Your ${listType} must only contain valid objects. Found '${data}'`,
+ };
+ }
+ if (!Object.hasOwn(data, referenceProperty)) {
+ return {
+ isValid: false,
+ message: `The reference property '${String(referenceProperty)}' is not available in all the objects of your ${listType}.`,
+ };
+ }
+ return {
+ isValid: true,
+ message: "",
+ };
+}
+
+export function outputDiffChunk<T extends Record<string, unknown>>(
+ emitter: Emitter<T>,
+) {
+ let chunks: StreamListDiff<T>[] = [];
+
+ function handleDiffChunk(
+ chunk: StreamListDiff<T>,
+ options: ListStreamOptions,
+ ): void {
+ const showChunk = options?.showOnly
+ ? options?.showOnly.includes(chunk.status)
+ : true;
+ if (!showChunk) {
+ return;
+ }
+ if ((options.chunksSize as number) > 0) {
+ chunks.push(chunk);
+ if (chunks.length >= (options.chunksSize as number)) {
+ const output = chunks;
+ chunks = [];
+ return emitter.emit(StreamEvent.Data, output);
+ } else {
+ return;
+ }
+ }
+ return emitter.emit(StreamEvent.Data, [chunk]);
+ }
+
+ function releaseLastChunks() {
+ if (chunks.length > 0) {
+ const output = chunks;
+ chunks = [];
+ return emitter.emit(StreamEvent.Data, output);
+ }
+ }
+
+ return {
+ handleDiffChunk,
+ releaseLastChunks,
+ };
+}
diff --git a/src/lib/utils/index.ts b/src/lib/utils/index.ts
index 1e26b91..435fece 100644
--- a/src/lib/utils/index.ts
+++ b/src/lib/utils/index.ts
@@ -38,3 +38,5 @@ export function isEqual(
export function isObject(value: unknown): value is Record<string, unknown> {
return !!value && typeof value === "object" && !Array.isArray(value);
}
+
+export const isClient = () => typeof window !== "undefined";
diff --git a/src/mocks/nextList.json b/src/mocks/nextList.json
new file mode 100644
index 0000000..942a1d5
--- /dev/null
+++ b/src/mocks/nextList.json
@@ -0,0 +1,6 @@
+[
+ { "id": 1, "name": "Item 1" },
+ { "id": 2, "name": "Item Two" },
+ { "id": 3, "name": "Item 3" },
+ { "id": 5, "name": "Item 5" }
+]
\ No newline at end of file
diff --git a/src/mocks/prevList.json b/src/mocks/prevList.json
new file mode 100644
index 0000000..00502da
--- /dev/null
+++ b/src/mocks/prevList.json
@@ -0,0 +1,6 @@
+[
+ { "id": 1, "name": "Item 1" },
+ { "id": 2, "name": "Item 2" },
+ { "id": 3, "name": "Item 3" },
+ { "id": 4, "name": "Item 4" }
+ ]
\ No newline at end of file
diff --git a/src/models/stream/index.ts b/src/models/stream/index.ts
index d5d0df4..f62b50d 100644
--- a/src/models/stream/index.ts
+++ b/src/models/stream/index.ts
@@ -16,6 +16,14 @@ export type StreamReferences<T extends Record<string, unknown>> = Map<
{ prevIndex: number; nextIndex?: number }
>;
+export type DataBuffer<T extends Record<string, unknown>> = Map<
+ ReferenceProperty<T>,
+ {
+ data: T | null;
+ index: number | null;
+ }
+>;
+
export type ListStreamOptions = {
chunksSize?: number; // 0 by default.
showOnly?: `${LIST_STATUS}`[];
@@ -25,3 +33,5 @@ export type ListStreamOptions = {
export const DEFAULT_LIST_STREAM_OPTIONS: ListStreamOptions = {
chunksSize: 0,
};
+
+export type FilePath = string;