From ffc1ad3e84494bcb3bac428b0a67a9e32c006c17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luc=C3=A0s?= <86352901+LucasVbr@users.noreply.github.com> Date: Sat, 28 Sep 2024 16:19:43 +0200 Subject: [PATCH] feat: Stream download, extract and parse datasets --- package-lock.json | 801 +++++------------- package.json | 10 +- src/index.ts | 9 +- src/services/CacheService.ts | 25 +- src/services/DMN.ts | 10 + src/services/FileService.ts | 62 +- src/services/archive/Archive.ts | 7 + src/services/archive/ArchiveFactory.ts | 12 + src/services/archive/ArchiveType.ts | 7 + src/services/archive/GzipArchive.ts | 12 + src/services/archive/ZipArchive.ts | 15 + src/services/archive/index.ts | 7 + .../archive_extractor/ArchiveExtractor.ts | 5 - .../ArchiveExtractorFactory.ts | 16 - .../archive_extractor/ZipExtractor.ts | 18 - src/services/archive_extractor/index.ts | 6 - src/services/data/Data.ts | 3 + src/services/data/NudgerData.ts | 28 + src/services/data/index.ts | 3 + src/services/dataset/Dataset.ts | 76 ++ src/services/dataset/DatasetCollection.ts | 20 + src/services/dataset/DatasetType.ts | 10 + src/services/dataset/NudgerDatasetService.ts | 40 - src/services/dataset/index.ts | 3 + src/services/dataset_parser/CsvParser.ts | 17 - src/services/dataset_parser/DatasetParser.ts | 5 - .../dataset_parser/DatasetParserFactory.ts | 16 - src/services/dataset_parser/index.ts | 6 - src/services/parser/CsvParser.ts | 13 + src/services/parser/Parser.ts | 10 + src/services/parser/ParserFactory.ts | 16 + src/services/parser/index.ts | 4 + 32 files changed, 501 insertions(+), 791 deletions(-) create mode 100644 src/services/DMN.ts create mode 100644 src/services/archive/Archive.ts create mode 100644 src/services/archive/ArchiveFactory.ts create mode 100644 src/services/archive/ArchiveType.ts create mode 100644 src/services/archive/GzipArchive.ts create mode 100644 src/services/archive/ZipArchive.ts create mode 100644 src/services/archive/index.ts delete mode 100644 src/services/archive_extractor/ArchiveExtractor.ts delete mode 100644 src/services/archive_extractor/ArchiveExtractorFactory.ts delete mode 100644 src/services/archive_extractor/ZipExtractor.ts delete mode 100644 src/services/archive_extractor/index.ts create mode 100644 src/services/data/Data.ts create mode 100644 src/services/data/NudgerData.ts create mode 100644 src/services/data/index.ts create mode 100644 src/services/dataset/Dataset.ts create mode 100644 src/services/dataset/DatasetCollection.ts create mode 100644 src/services/dataset/DatasetType.ts delete mode 100644 src/services/dataset/NudgerDatasetService.ts create mode 100644 src/services/dataset/index.ts delete mode 100644 src/services/dataset_parser/CsvParser.ts delete mode 100644 src/services/dataset_parser/DatasetParser.ts delete mode 100644 src/services/dataset_parser/DatasetParserFactory.ts delete mode 100644 src/services/dataset_parser/index.ts create mode 100644 src/services/parser/CsvParser.ts create mode 100644 src/services/parser/Parser.ts create mode 100644 src/services/parser/ParserFactory.ts create mode 100644 src/services/parser/index.ts diff --git a/package-lock.json b/package-lock.json index c8e36cb..e1d4604 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,146 +9,24 @@ "version": "1.0.0", "license": "ISC", "dependencies": { - "dmn-js": "^16.7.1", + "axios": "^1.7.7", + "csvtojson": "^2.0.10", "dotenv": "^16.4.5", "express": "^4.21.0", - "fs-extra": "^11.2.0", - "papaparse": "^5.4.1", + "node-stream-zip": "^1.15.0", + "tar-stream": "^3.1.7", "unzipper": "^0.12.3" }, "devDependencies": { "@types/express": "^4.17.21", - "@types/fs-extra": "^11.0.4", "@types/node": "^22.5.5", - "@types/papaparse": "^5.3.14", + "@types/tar-stream": "^3.1.3", "@types/unzipper": "^0.10.10", "prettier": "3.3.3", "ts-node": "^10.9.2", "typescript": "^5.6.2" } }, - "node_modules/@bpmn-io/diagram-js-ui": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@bpmn-io/diagram-js-ui/-/diagram-js-ui-0.2.3.tgz", - "integrity": "sha512-OGyjZKvGK8tHSZ0l7RfeKhilGoOGtFDcoqSGYkX0uhFlo99OVZ9Jn1K7TJGzcE9BdKwvA5Y5kGqHEhdTxHvFfw==", - "license": "MIT", - "dependencies": { - "htm": "^3.1.1", - "preact": "^10.11.2" - } - }, - "node_modules/@bpmn-io/dmn-variable-resolver": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/@bpmn-io/dmn-variable-resolver/-/dmn-variable-resolver-0.7.0.tgz", - "integrity": "sha512-ssL8fch5U0q8efbrHdgSGznh5Dlk+R0MNnazCtDpyac6yqlgn/JQ/HewCjacTk4XFubbMiDbuEIESHLBsWCBxg==", - "license": "MIT" - }, - "node_modules/@bpmn-io/feel-editor": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@bpmn-io/feel-editor/-/feel-editor-1.9.0.tgz", - "integrity": "sha512-O9w+UbajbxenLukbQGcWAfg91zV2FCJi5ba+nD3NN9pwyF0CL0m8Vf2GhJuALlOsJshRW6dJSw/TU0NMax+B7A==", - "license": "MIT", - "dependencies": { - "@bpmn-io/feel-lint": "^1.3.0", - "@codemirror/autocomplete": "^6.16.2", - "@codemirror/commands": "^6.6.0", - "@codemirror/language": "^6.10.2", - "@codemirror/lint": "^6.8.0", - "@codemirror/state": "^6.4.1", - "@codemirror/view": "^6.28.1", - "@lezer/highlight": "^1.2.0", - "lang-feel": "^2.1.1", - "min-dom": "^4.2.1" - }, - "engines": { - "node": ">= 16" - } - }, - "node_modules/@bpmn-io/feel-lint": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/@bpmn-io/feel-lint/-/feel-lint-1.3.1.tgz", - "integrity": "sha512-wcFkJKhOm/iqCt5bzkKvxL5Dr9wKwUD+t164bQYbJsTYouAqmkkxiGsoqck42hXwdIhMSguZ+vqQ3hj5QdiYCA==", - "license": "MIT", - "dependencies": { - "@codemirror/language": "^6.10.0", - "lezer-feel": "^1.2.3" - }, - "engines": { - "node": "*" - } - }, - "node_modules/@codemirror/autocomplete": { - "version": "6.18.1", - "resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-6.18.1.tgz", - "integrity": "sha512-iWHdj/B1ethnHRTwZj+C1obmmuCzquH29EbcKr0qIjA9NfDeBDJ7vs+WOHsFeLeflE4o+dHfYndJloMKHUkWUA==", - "license": "MIT", - "dependencies": { - "@codemirror/language": "^6.0.0", - "@codemirror/state": "^6.0.0", - "@codemirror/view": "^6.17.0", - "@lezer/common": "^1.0.0" - }, - "peerDependencies": { - "@codemirror/language": "^6.0.0", - "@codemirror/state": "^6.0.0", - "@codemirror/view": "^6.0.0", - "@lezer/common": "^1.0.0" - } - }, - "node_modules/@codemirror/commands": { - "version": "6.6.2", - "resolved": "https://registry.npmjs.org/@codemirror/commands/-/commands-6.6.2.tgz", - "integrity": "sha512-Fq7eWOl1Rcbrfn6jD8FPCj9Auaxdm5nIK5RYOeW7ughnd/rY5AmPg6b+CfsG39ZHdwiwe8lde3q8uR7CF5S0yQ==", - "license": "MIT", - "dependencies": { - "@codemirror/language": "^6.0.0", - "@codemirror/state": "^6.4.0", - "@codemirror/view": "^6.27.0", - "@lezer/common": "^1.1.0" - } - }, - "node_modules/@codemirror/language": { - "version": "6.10.3", - "resolved": "https://registry.npmjs.org/@codemirror/language/-/language-6.10.3.tgz", - "integrity": "sha512-kDqEU5sCP55Oabl6E7m5N+vZRoc0iWqgDVhEKifcHzPzjqCegcO4amfrYVL9PmPZpl4G0yjkpTpUO/Ui8CzO8A==", - "license": "MIT", - "dependencies": { - "@codemirror/state": "^6.0.0", - "@codemirror/view": "^6.23.0", - "@lezer/common": "^1.1.0", - "@lezer/highlight": "^1.0.0", - "@lezer/lr": "^1.0.0", - "style-mod": "^4.0.0" - } - }, - "node_modules/@codemirror/lint": { - "version": "6.8.1", - "resolved": "https://registry.npmjs.org/@codemirror/lint/-/lint-6.8.1.tgz", - "integrity": "sha512-IZ0Y7S4/bpaunwggW2jYqwLuHj0QtESf5xcROewY6+lDNwZ/NzvR4t+vpYgg9m7V8UXLPYqG+lu3DF470E5Oxg==", - "license": "MIT", - "dependencies": { - "@codemirror/state": "^6.0.0", - "@codemirror/view": "^6.0.0", - "crelt": "^1.0.5" - } - }, - "node_modules/@codemirror/state": { - "version": "6.4.1", - "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.4.1.tgz", - "integrity": "sha512-QkEyUiLhsJoZkbumGZlswmAhA7CBU02Wrz7zvH4SrcifbsqwlXShVXg65f3v/ts57W3dqyamEriMhij1Z3Zz4A==", - "license": "MIT" - }, - "node_modules/@codemirror/view": { - "version": "6.33.0", - "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.33.0.tgz", - "integrity": "sha512-AroaR3BvnjRW8fiZBalAaK+ZzB5usGgI014YKElYZvQdNH5ZIidHlO+cyf/2rWzyBFRkvG6VhiXeAEbC53P2YQ==", - "license": "MIT", - "dependencies": { - "@codemirror/state": "^6.4.0", - "style-mod": "^4.1.0", - "w3c-keyname": "^2.2.4" - } - }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -190,30 +68,6 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, - "node_modules/@lezer/common": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.2.1.tgz", - "integrity": "sha512-yemX0ZD2xS/73llMZIK6KplkjIjf2EvAHcinDi/TfJ9hS25G0388+ClHt6/3but0oOxinTcQHJLDXh6w1crzFQ==", - "license": "MIT" - }, - "node_modules/@lezer/highlight": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-1.2.1.tgz", - "integrity": "sha512-Z5duk4RN/3zuVO7Jq0pGLJ3qynpxUVsh7IbUbGj88+uV2ApSAn6kWg2au3iJb+0Zi7kKtqffIESgNcRXWZWmSA==", - "license": "MIT", - "dependencies": { - "@lezer/common": "^1.0.0" - } - }, - "node_modules/@lezer/lr": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.4.2.tgz", - "integrity": "sha512-pu0K1jCIdnQ12aWNaAVU5bzi7Bd1w54J3ECgANPmYLtQKP0HBj2cE/5coBD66MT10xbtIuUr7tg0Shbsvk0mDA==", - "license": "MIT", - "dependencies": { - "@lezer/common": "^1.0.0" - } - }, "node_modules/@tsconfig/node10": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", @@ -289,17 +143,6 @@ "@types/send": "*" } }, - "node_modules/@types/fs-extra": { - "version": "11.0.4", - "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-11.0.4.tgz", - "integrity": "sha512-yTbItCNreRooED33qjunPthRcSjERP1r4MqCZc7wv0u2sUkzTFp45tgUfS5+r7FrZPdmCCNflLhVSP/o+SemsQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/jsonfile": "*", - "@types/node": "*" - } - }, "node_modules/@types/http-errors": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz", @@ -307,16 +150,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/jsonfile": { - "version": "6.1.4", - "resolved": "https://registry.npmjs.org/@types/jsonfile/-/jsonfile-6.1.4.tgz", - "integrity": "sha512-D5qGUYwjvnNNextdU59/+fI+spnwtTFmyQP0h+PfIOSkNfpU6AOICUOkm4i0OnSk+NyjdPJrxCDro0sJsWlRpQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/mime": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", @@ -334,16 +167,6 @@ "undici-types": "~6.19.2" } }, - "node_modules/@types/papaparse": { - "version": "5.3.14", - "resolved": "https://registry.npmjs.org/@types/papaparse/-/papaparse-5.3.14.tgz", - "integrity": "sha512-LxJ4iEFcpqc6METwp9f6BV6VVc43m6MfH0VqFosHvrUgfXiFe6ww7R3itkOQ+TCK6Y+Iv/+RnnvtRZnkc5Kc9g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/qs": { "version": "6.9.16", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.16.tgz", @@ -381,6 +204,16 @@ "@types/send": "*" } }, + "node_modules/@types/tar-stream": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@types/tar-stream/-/tar-stream-3.1.3.tgz", + "integrity": "sha512-Zbnx4wpkWBMBSu5CytMbrT5ZpMiF55qgM+EpHzR4yIDu7mv52cej8hTkOc6K+LzpkOAbxwn/m7j3iO+/l42YkQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/unzipper": { "version": "0.10.10", "resolved": "https://registry.npmjs.org/@types/unzipper/-/unzipper-0.10.10.tgz", @@ -443,6 +276,36 @@ "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", "license": "MIT" }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.7.7", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", + "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/b4a": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.7.tgz", + "integrity": "sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==", + "license": "Apache-2.0" + }, + "node_modules/bare-events": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.5.0.tgz", + "integrity": "sha512-/E8dDe9dsbLyh2qrZ64PEPadOQ0F4gbl1sUJOrmph7xOiIxfY8vwab/4bFLh4Y88/Hk/ujKcrQKc+ps0mv873A==", + "license": "Apache-2.0", + "optional": true + }, "node_modules/bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", @@ -501,32 +364,18 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/clsx": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", - "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, "engines": { - "node": ">=6" + "node": ">= 0.8" } }, - "node_modules/component-event": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/component-event/-/component-event-0.2.1.tgz", - "integrity": "sha512-wGA++isMqiDq1jPYeyv2as/Bt/u+3iLW0rEa+8NQ82jAv3TgqMiCM+B2SaBdn2DfLilLjjq736YcezihRYhfxw==", - "license": "MIT" - }, - "node_modules/component-props": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/component-props/-/component-props-1.1.1.tgz", - "integrity": "sha512-69pIRJs9fCCHRqCz3390YF2LV1Lu6iEMZ5zuVqqUn+G20V9BNXlMs0cWawWeW9g4Ynmg29JmkG6R7/lUJoGd1Q==" - }, - "node_modules/component-xor": { - "version": "0.0.4", - "resolved": "https://registry.npmjs.org/component-xor/-/component-xor-0.0.4.tgz", - "integrity": "sha512-ZIt6sla8gfo+AFVRZoZOertcnD5LJaY2T9CKE2j13NJxQt/mUafD69Bl7/Y4AnpI2LGjiXH7cOfJDx/n2G9edA==", - "license": "MIT" - }, "node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -576,17 +425,22 @@ "dev": true, "license": "MIT" }, - "node_modules/crelt": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.6.tgz", - "integrity": "sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g==", - "license": "MIT" - }, - "node_modules/css.escape": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/css.escape/-/css.escape-1.5.1.tgz", - "integrity": "sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg==", - "license": "MIT" + "node_modules/csvtojson": { + "version": "2.0.10", + "resolved": "https://registry.npmjs.org/csvtojson/-/csvtojson-2.0.10.tgz", + "integrity": "sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ==", + "license": "MIT", + "dependencies": { + "bluebird": "^3.5.1", + "lodash": "^4.17.3", + "strip-bom": "^2.0.0" + }, + "bin": { + "csvtojson": "bin/csvtojson" + }, + "engines": { + "node": ">=4.0.0" + } }, "node_modules/debug": { "version": "2.6.9", @@ -614,6 +468,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -633,51 +496,6 @@ "npm": "1.2.8000 || >= 1.4.16" } }, - "node_modules/diagram-js": { - "version": "14.11.0", - "resolved": "https://registry.npmjs.org/diagram-js/-/diagram-js-14.11.0.tgz", - "integrity": "sha512-OzJXMj9xPrVMgrvFNbr7ml6vcOrnt/FYO4at0SRY5B6EnU4nlqfncQx1VNYcA97/eIh/prAc0SxhqyZkSKivRw==", - "license": "MIT", - "dependencies": { - "@bpmn-io/diagram-js-ui": "^0.2.3", - "clsx": "^2.1.0", - "didi": "^10.2.2", - "inherits-browser": "^0.1.0", - "min-dash": "^4.1.0", - "min-dom": "^4.2.1", - "object-refs": "^0.4.0", - "path-intersection": "^3.0.0", - "tiny-svg": "^3.1.2" - }, - "engines": { - "node": "*" - } - }, - "node_modules/diagram-js-direct-editing": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/diagram-js-direct-editing/-/diagram-js-direct-editing-3.1.0.tgz", - "integrity": "sha512-rBo60hhwUT7XwB1v8UEHHwLgcoz/KmeTKkVGWwkWz9/56XA7EvJkg9UDL758eon1IF6HbSkpk0oZm/qLl7Nvfg==", - "license": "MIT", - "dependencies": { - "min-dash": "^4.0.0", - "min-dom": "^4.2.1" - }, - "engines": { - "node": "*" - }, - "peerDependencies": { - "diagram-js": "*" - } - }, - "node_modules/didi": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/didi/-/didi-10.2.2.tgz", - "integrity": "sha512-l8NYkYFXV1izHI65EyT8EXOjUZtKmQkHLTT89cSP7HU5J/G7AOj0dXKtLc04EXYlga99PBY18IPjOeZ+c3DI4w==", - "license": "MIT", - "engines": { - "node": ">= 16" - } - }, "node_modules/diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -688,139 +506,6 @@ "node": ">=0.3.1" } }, - "node_modules/dmn-js": { - "version": "16.7.1", - "resolved": "https://registry.npmjs.org/dmn-js/-/dmn-js-16.7.1.tgz", - "integrity": "sha512-TPlMNFsuR9Px665IyR/Z1zNinQ7PNO8kFEdgLCCysfEdXTWR5vrMYCSWg9GkRZ7D47EJJgnVoBFA8setjbZeQg==", - "license": "SEE LICENSE IN LICENSE", - "dependencies": { - "dmn-js-boxed-expression": "^16.7.1", - "dmn-js-decision-table": "^16.7.1", - "dmn-js-drd": "^16.7.1", - "dmn-js-literal-expression": "^16.7.1", - "dmn-js-shared": "^16.7.1" - } - }, - "node_modules/dmn-js-boxed-expression": { - "version": "16.7.1", - "resolved": "https://registry.npmjs.org/dmn-js-boxed-expression/-/dmn-js-boxed-expression-16.7.1.tgz", - "integrity": "sha512-/u7axnD7aPDJkTT2o2aoMLsQJnO8Qo5IEwcXcA+Z3U1mjDU2rosSj/eknmQr6mLBiYatPeWi1sbyYqF3cnu2fw==", - "license": "SEE LICENSE IN LICENSE", - "dependencies": { - "@bpmn-io/dmn-variable-resolver": "^0.7.0", - "diagram-js": "^14.7.1", - "dmn-js-shared": "^16.7.1", - "inferno": "~5.6.3", - "min-dash": "^4.0.0", - "min-dom": "^4.2.1", - "table-js": "^9.1.0" - } - }, - "node_modules/dmn-js-decision-table": { - "version": "16.7.1", - "resolved": "https://registry.npmjs.org/dmn-js-decision-table/-/dmn-js-decision-table-16.7.1.tgz", - "integrity": "sha512-2Xkbc/zxMTduMBMZ1lsr5qktFudGYjmrTxR0WEImfEUjKnVqIZHRSWcY8aGEfTMtAdHtYLsldG9vozZfHqmTRQ==", - "license": "SEE LICENSE IN LICENSE", - "dependencies": { - "@bpmn-io/dmn-variable-resolver": "^0.7.0", - "css.escape": "^1.5.1", - "diagram-js": "^14.7.1", - "dmn-js-shared": "^16.7.1", - "escape-html": "^1.0.3", - "inferno": "~5.6.3", - "min-dash": "^4.0.0", - "min-dom": "^4.2.1", - "selection-ranges": "^3.0.2", - "table-js": "^9.1.0" - } - }, - "node_modules/dmn-js-drd": { - "version": "16.7.1", - "resolved": "https://registry.npmjs.org/dmn-js-drd/-/dmn-js-drd-16.7.1.tgz", - "integrity": "sha512-lU0zHqH+gOiMXtXxw7I1LCCa84nGKDiIOIgqBvYVT26oqZd/a6w4rsfbIhFYM4TEtXPqFqo+hzRTPqmXg0zpeA==", - "license": "SEE LICENSE IN LICENSE", - "dependencies": { - "diagram-js": "^14.7.1", - "diagram-js-direct-editing": "^3.0.1", - "dmn-js-shared": "^16.7.1", - "inherits-browser": "^0.1.0", - "min-dash": "^4.0.0", - "min-dom": "^4.2.1", - "object-refs": "^0.4.0", - "tiny-svg": "^3.1.2" - } - }, - "node_modules/dmn-js-literal-expression": { - "version": "16.7.1", - "resolved": "https://registry.npmjs.org/dmn-js-literal-expression/-/dmn-js-literal-expression-16.7.1.tgz", - "integrity": "sha512-k6mP1ubztixZGM2N2W3bHeFZ677b2yU+uPi2GECRfttnl+j+XxCkCylvY1xtbewjs7lV9sX/d90lAXEK9H9VQg==", - "license": "SEE LICENSE IN LICENSE", - "dependencies": { - "@bpmn-io/dmn-variable-resolver": "^0.7.0", - "diagram-js": "^14.7.1", - "dmn-js-shared": "^16.7.1", - "escape-html": "^1.0.3", - "inferno": "~5.6.3", - "min-dash": "^4.0.0", - "min-dom": "^4.2.1", - "table-js": "^9.1.0" - } - }, - "node_modules/dmn-js-shared": { - "version": "16.7.1", - "resolved": "https://registry.npmjs.org/dmn-js-shared/-/dmn-js-shared-16.7.1.tgz", - "integrity": "sha512-pX9JI1PRTHO1j+RccvpFaYN0zXoSpAgXnfHGTQbgfmnKGvJ8DAxU6V9aulRKs8+ORn//hczTlTD+25Z9Zq7XeA==", - "license": "SEE LICENSE IN LICENSE", - "dependencies": { - "@bpmn-io/feel-editor": "^1.8.0", - "diagram-js": "^14.7.1", - "didi": "^10.2.2", - "dmn-moddle": "^10.0.0", - "ids": "^1.0.5", - "inferno": "~5.6.3", - "min-dash": "^4.0.0", - "min-dom": "^4.2.1", - "selection-ranges": "^3.0.2", - "selection-update": "^0.1.2", - "table-js": "^9.1.0" - } - }, - "node_modules/dmn-moddle": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/dmn-moddle/-/dmn-moddle-10.0.0.tgz", - "integrity": "sha512-JD08qH2VqA7O54qCQFrGruwGyVovow+9g2O5/ww0KpC/n0mvuua117dz2CONiUWexJxq/dOAaMjrQwkWnK+oEA==", - "license": "MIT", - "dependencies": { - "min-dash": "^3.0.0", - "moddle": "^5.0.1", - "moddle-xml": "^9.0.5" - } - }, - "node_modules/dmn-moddle/node_modules/min-dash": { - "version": "3.8.1", - "resolved": "https://registry.npmjs.org/min-dash/-/min-dash-3.8.1.tgz", - "integrity": "sha512-evumdlmIlg9mbRVPbC4F5FuRhNmcMS5pvuBUbqb1G9v09Ro0ImPEgz5n3khir83lFok1inKqVDjnKEg3GpDxQg==", - "license": "MIT" - }, - "node_modules/dom-iterator": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/dom-iterator/-/dom-iterator-1.0.0.tgz", - "integrity": "sha512-7dsMOQI07EMU98gQM8NSB3GsAiIeBYIPKpnxR3c9xOvdvBjChAcOM0iJ222I3p5xyiZO9e5oggkNaCusuTdYig==", - "license": "MIT", - "dependencies": { - "component-props": "1.1.1", - "component-xor": "0.0.4" - } - }, - "node_modules/domify": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/domify/-/domify-1.4.2.tgz", - "integrity": "sha512-m4yreHcUWHBncGVV7U+yQzc12vIlq0jMrtHZ5mW6dQMiL/7skSYNVX9wqKwOtyO9SGCgevrAFEgOCAHmamHTUA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/dotenv": { "version": "16.4.5", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", @@ -935,6 +620,12 @@ "node": ">= 0.10.0" } }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, "node_modules/finalhandler": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", @@ -953,6 +644,40 @@ "node": ">= 0.8" } }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -1079,12 +804,6 @@ "node": ">= 0.4" } }, - "node_modules/htm": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/htm/-/htm-3.1.1.tgz", - "integrity": "sha512-983Vyg8NwUE7JkZ6NmOqpCZ+sh1bKv2iYTlUkzlWmA5JD2acKoxd4KVxbMmxX/85mtfdnDmTFoNKcg5DGAvxNQ==", - "license": "Apache-2.0" - }, "node_modules/http-errors": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", @@ -1113,48 +832,12 @@ "node": ">=0.10.0" } }, - "node_modules/ids": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/ids/-/ids-1.0.5.tgz", - "integrity": "sha512-XQ0yom/4KWTL29sLG+tyuycy7UmeaM/79GRtSJq6IG9cJGIPeBz5kwDCguie3TwxaMNIc3WtPi0cTa1XYHicpw==", - "license": "MIT" - }, - "node_modules/inferno": { - "version": "5.6.3", - "resolved": "https://registry.npmjs.org/inferno/-/inferno-5.6.3.tgz", - "integrity": "sha512-2m7bEo/6D+pbxFkFKY/2QOVdknxxXyrXyDdVFobJcQyvfv3985MY+VVwyRZoVJylQG+mzPY5/uDNuXygp8jObA==", - "hasInstallScript": true, - "license": "MIT", - "dependencies": { - "inferno-shared": "5.6.3", - "inferno-vnode-flags": "5.6.3", - "opencollective-postinstall": "^2.0.3" - } - }, - "node_modules/inferno-shared": { - "version": "5.6.3", - "resolved": "https://registry.npmjs.org/inferno-shared/-/inferno-shared-5.6.3.tgz", - "integrity": "sha512-NzhS62Y6BwfkLNmXS9C0Ie7skY1vIoLrVIYnVrMx1xfTLp/AG+ths4oAJGmWcX/AkVSYl+DjP96HEuUdtGmRRw==", - "license": "MIT" - }, - "node_modules/inferno-vnode-flags": { - "version": "5.6.3", - "resolved": "https://registry.npmjs.org/inferno-vnode-flags/-/inferno-vnode-flags-5.6.3.tgz", - "integrity": "sha512-1V9Pjz5Erm2sP7xlb4D2zW/U8e3RwU41Ruyn9/apke0BFxbLI+LxxVcKWxom4fPetG6YcBNLpZssy8gkigT9gg==", - "license": "MIT" - }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "license": "ISC" }, - "node_modules/inherits-browser": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/inherits-browser/-/inherits-browser-0.1.0.tgz", - "integrity": "sha512-CJHHvW3jQ6q7lzsXPpapLdMx5hDpSF3FSh45pwsj6bKxJJ8Nl8v43i5yXnr3BdfOimGHKyniewQtnAIp3vyJJw==", - "license": "ISC" - }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -1164,6 +847,12 @@ "node": ">= 0.10" } }, + "node_modules/is-utf8": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz", + "integrity": "sha512-rMYPYvCzsXywIsldgLaSoPlw5PfoB/ssr7hY4pLfcodrA5M/eArza1a9VmTiNIBNMjOGr1Ow9mTyU2o69U6U9Q==", + "license": "MIT" + }, "node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", @@ -1182,35 +871,11 @@ "graceful-fs": "^4.1.6" } }, - "node_modules/lang-feel": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/lang-feel/-/lang-feel-2.2.0.tgz", - "integrity": "sha512-Ebo5nftYsMfJzB3Ny8Oy4oaDXZXb5x61qtVVmKv6aImvAZUbT76mD60ZbEilizjZQzsR2CcU1iMK5sacIa1NVA==", - "license": "MIT", - "dependencies": { - "@codemirror/autocomplete": "^6.16.2", - "@codemirror/language": "^6.10.2", - "@codemirror/state": "^6.4.1", - "@codemirror/view": "^6.28.1", - "@lezer/common": "^1.2.1", - "lezer-feel": "^1.3.0" - }, - "engines": { - "node": "*" - } - }, - "node_modules/lezer-feel": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/lezer-feel/-/lezer-feel-1.3.0.tgz", - "integrity": "sha512-b7oBzdbcdHJU+WIVwqELi/pLLg0UiojC/80E2zFBGrZsWTSOsSQLNhHsfRocYzWLUPpOC4pWaFlhJM4m93J5Fg==", - "license": "MIT", - "dependencies": { - "@lezer/highlight": "^1.2.0", - "@lezer/lr": "^1.4.1" - }, - "engines": { - "node": "*" - } + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "license": "MIT" }, "node_modules/make-error": { "version": "1.3.6", @@ -1279,55 +944,6 @@ "node": ">= 0.6" } }, - "node_modules/min-dash": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/min-dash/-/min-dash-4.2.1.tgz", - "integrity": "sha512-to+unsToePnm7cUeR9TrMzFlETHd/UXmU+ELTRfWZj5XGT41KF6X3L233o3E/GdEs3sk2Tbw/lOLD1avmWkg8A==", - "license": "MIT" - }, - "node_modules/min-dom": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/min-dom/-/min-dom-4.2.1.tgz", - "integrity": "sha512-TMoL8SEEIhUWYgkj7XMSgxmwSyGI+4fP2KFFGnN3FbHfbGHVdsLYSz8LoIsgPhz4dWRmLvxWWSMgzZMJW5sZuA==", - "license": "MIT", - "dependencies": { - "component-event": "^0.2.1", - "domify": "^1.4.1", - "min-dash": "^4.2.1" - } - }, - "node_modules/moddle": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/moddle/-/moddle-5.0.4.tgz", - "integrity": "sha512-Kjb+hjuzO+YlojNGxEUXvdhLYTHTtAABDlDcJTtTcn5MbJF9Zkv4I1Fyvp3Ypmfgg1EfHDZ3PsCQTuML9JD6wg==", - "license": "MIT", - "dependencies": { - "min-dash": "^3.0.0" - } - }, - "node_modules/moddle-xml": { - "version": "9.0.6", - "resolved": "https://registry.npmjs.org/moddle-xml/-/moddle-xml-9.0.6.tgz", - "integrity": "sha512-tl0reHpsY/aKlLGhXeFlQWlYAQHFxTkFqC8tq8jXRYpQSnLVw13T6swMaourLd7EXqHdWsc+5ggsB+fEep6xZQ==", - "license": "MIT", - "dependencies": { - "min-dash": "^3.5.2", - "moddle": "^5.0.2", - "saxen": "^8.1.2" - } - }, - "node_modules/moddle-xml/node_modules/min-dash": { - "version": "3.8.1", - "resolved": "https://registry.npmjs.org/min-dash/-/min-dash-3.8.1.tgz", - "integrity": "sha512-evumdlmIlg9mbRVPbC4F5FuRhNmcMS5pvuBUbqb1G9v09Ro0ImPEgz5n3khir83lFok1inKqVDjnKEg3GpDxQg==", - "license": "MIT" - }, - "node_modules/moddle/node_modules/min-dash": { - "version": "3.8.1", - "resolved": "https://registry.npmjs.org/min-dash/-/min-dash-3.8.1.tgz", - "integrity": "sha512-evumdlmIlg9mbRVPbC4F5FuRhNmcMS5pvuBUbqb1G9v09Ro0ImPEgz5n3khir83lFok1inKqVDjnKEg3GpDxQg==", - "license": "MIT" - }, "node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -1349,6 +965,19 @@ "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", "license": "MIT" }, + "node_modules/node-stream-zip": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.15.0.tgz", + "integrity": "sha512-LN4fydt9TqhZhThkZIVQnF9cwjU3qmUH9h78Mx/K7d3VvfRqqwthLwJEUOEL0QPZ0XQmNN7be5Ggit5+4dq3Bw==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/antelle" + } + }, "node_modules/object-inspect": { "version": "1.13.2", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", @@ -1361,15 +990,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/object-refs": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/object-refs/-/object-refs-0.4.0.tgz", - "integrity": "sha512-6kJqKWryKZmtte6QYvouas0/EIJKPI1/MMIuRsiBlNuhIMfqYTggzX2F1AJ2+cDs288xyi9GL7FyasHINR98BQ==", - "license": "MIT", - "engines": { - "node": "*" - } - }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -1382,21 +1002,6 @@ "node": ">= 0.8" } }, - "node_modules/opencollective-postinstall": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz", - "integrity": "sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==", - "license": "MIT", - "bin": { - "opencollective-postinstall": "index.js" - } - }, - "node_modules/papaparse": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.4.1.tgz", - "integrity": "sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw==", - "license": "MIT" - }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -1406,31 +1011,12 @@ "node": ">= 0.8" } }, - "node_modules/path-intersection": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/path-intersection/-/path-intersection-3.1.0.tgz", - "integrity": "sha512-3xS3lvv/vuwm5aH2BVvNRvnvwR2Drde7jQClKpCXTYXIMMjcw/EnMhzCgeHwqbCpzi760PEfAkU53vSIlrNr9A==", - "license": "MIT", - "engines": { - "node": ">= 14.20" - } - }, "node_modules/path-to-regexp": { "version": "0.1.10", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz", "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==", "license": "MIT" }, - "node_modules/preact": { - "version": "10.24.0", - "resolved": "https://registry.npmjs.org/preact/-/preact-10.24.0.tgz", - "integrity": "sha512-aK8Cf+jkfyuZ0ZZRG9FbYqwmEiGQ4y/PUO4SuTWoyWL244nZZh7bd5h2APd4rSNDYTBNghg1L+5iJN3Skxtbsw==", - "license": "MIT", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/preact" - } - }, "node_modules/prettier": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.3.tgz", @@ -1466,6 +1052,12 @@ "node": ">= 0.10" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, "node_modules/qs": { "version": "6.13.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", @@ -1481,6 +1073,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/queue-tick": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", + "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==", + "license": "MIT" + }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -1552,27 +1150,6 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "license": "MIT" }, - "node_modules/saxen": { - "version": "8.1.2", - "resolved": "https://registry.npmjs.org/saxen/-/saxen-8.1.2.tgz", - "integrity": "sha512-xUOiiFbc3Ow7p8KMxwsGICPx46ZQvy3+qfNVhrkwfz3Vvq45eGt98Ft5IQaA1R/7Tb5B5MKh9fUR9x3c3nDTxw==", - "license": "MIT" - }, - "node_modules/selection-ranges": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/selection-ranges/-/selection-ranges-3.0.3.tgz", - "integrity": "sha512-60Oqc07j16YCrp96uITgBFu7oT81JKMmL+cOOcxe3jvuGSiFSwsLpOSXNBAlITV9hGhEl1H6P/+g1bKnpfXoSw==", - "license": "MIT", - "dependencies": { - "dom-iterator": "^1.0.0" - } - }, - "node_modules/selection-update": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/selection-update/-/selection-update-0.1.2.tgz", - "integrity": "sha512-4jzoJNh7VT2s2tvm/kUSskSw7pD0BVcrrGccbfOMK+3AXLBPz6nIy1yo+pbXgvNoTNII96Pq92+sAY+rF0LUAA==", - "license": "MIT" - }, "node_modules/send": { "version": "0.19.0", "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", @@ -1677,6 +1254,20 @@ "node": ">= 0.8" } }, + "node_modules/streamx": { + "version": "2.20.1", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.20.1.tgz", + "integrity": "sha512-uTa0mU6WUC65iUvzKH4X9hEdvSW7rbPxPtwfWiLMSj3qTdQbAiUboZTxauKfpFuGIGa1C2BYijZ7wgdUXICJhA==", + "license": "MIT", + "dependencies": { + "fast-fifo": "^1.3.2", + "queue-tick": "^1.0.1", + "text-decoder": "^1.1.0" + }, + "optionalDependencies": { + "bare-events": "^2.2.0" + } + }, "node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -1692,37 +1283,37 @@ "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", "license": "MIT" }, - "node_modules/style-mod": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/style-mod/-/style-mod-4.1.2.tgz", - "integrity": "sha512-wnD1HyVqpJUI2+eKZ+eo1UwghftP6yuFheBqqe+bWCotBjC2K1YnteJILRMs3SM4V/0dLEW1SC27MWP5y+mwmw==", - "license": "MIT" - }, - "node_modules/table-js": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/table-js/-/table-js-9.1.0.tgz", - "integrity": "sha512-yxzIBXZe5pFHXBng8+Rz3dDGiO4anWptlWQXP4Pm1oIetgdG4nq+DkZkRG4Lq5L1HFu1OVif4h3YMI7rxudvPw==", + "node_modules/strip-bom": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz", + "integrity": "sha512-kwrX1y7czp1E69n2ajbG65mIo9dqvJ+8aBQXOGVxqwvNbsXdFM6Lq37dLAY3mknUwru8CfcCbfOLL/gMo+fi3g==", "license": "MIT", "dependencies": { - "didi": "^10.0.0", - "ids": "^1.0.0", - "min-dash": "^4.0.0", - "min-dom": "^4.0.3", - "selection-ranges": "^3.0.3" + "is-utf8": "^0.2.0" }, "engines": { - "node": "*" - }, - "peerDependencies": { - "diagram-js": "^11.3.0 || ^12 || ^13.0.0 || ^14.0.0", - "inferno": "^5.0.5" + "node": ">=0.10.0" } }, - "node_modules/tiny-svg": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/tiny-svg/-/tiny-svg-3.1.2.tgz", - "integrity": "sha512-qvNlv/4N48HqbNpwAhzQ9HKHlBUKgA4091x+aVfsrRXHIcQ9NA3W6ZYwdmYAIdwT+vfAAksrc9L/3RQBj5KwPw==", - "license": "MIT" + "node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, + "node_modules/text-decoder": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.0.tgz", + "integrity": "sha512-n1yg1mOj9DNpk3NeZOx7T6jchTbyJS3i3cucbNN6FcdPriMZx7NsgrGpWWdWZZGxD7ES1XB+3uoqHMgOKaN+fg==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } }, "node_modules/toidentifier": { "version": "1.0.1", @@ -1873,12 +1464,6 @@ "node": ">= 0.8" } }, - "node_modules/w3c-keyname": { - "version": "2.2.8", - "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.8.tgz", - "integrity": "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==", - "license": "MIT" - }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", diff --git a/package.json b/package.json index bca2e66..0f2ca4b 100644 --- a/package.json +++ b/package.json @@ -15,18 +15,18 @@ "license": "ISC", "description": "", "dependencies": { - "dmn-js": "^16.7.1", + "axios": "^1.7.7", + "csvtojson": "^2.0.10", "dotenv": "^16.4.5", "express": "^4.21.0", - "fs-extra": "^11.2.0", - "papaparse": "^5.4.1", + "node-stream-zip": "^1.15.0", + "tar-stream": "^3.1.7", "unzipper": "^0.12.3" }, "devDependencies": { "@types/express": "^4.17.21", - "@types/fs-extra": "^11.0.4", "@types/node": "^22.5.5", - "@types/papaparse": "^5.3.14", + "@types/tar-stream": "^3.1.3", "@types/unzipper": "^0.10.10", "prettier": "3.3.3", "ts-node": "^10.9.2", diff --git a/src/index.ts b/src/index.ts index 787b1fb..8ef04a7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,9 +1,10 @@ import dotenv from "dotenv"; import Server from "./Server"; -import NudgerDatasetService from "./services/dataset/NudgerDatasetService"; +import { DatasetCollection } from "./services/dataset"; dotenv.config(); -Promise.all([NudgerDatasetService.loadDataset()]) - .then(() => new Server().start()) - .catch(console.error); +DatasetCollection.loadAll() + .then(() => console.log("All datasets are loaded")) + .then(() => new Server().start()) + .catch(console.error); diff --git a/src/services/CacheService.ts b/src/services/CacheService.ts index a76bfe0..447f150 100644 --- a/src/services/CacheService.ts +++ b/src/services/CacheService.ts @@ -1,27 +1,30 @@ import { createHash } from "node:crypto"; import { join } from "node:path"; -import * as fs from "fs-extra"; +import { existsSync, mkdirSync } from "node:fs"; + class CacheService { public static readonly CACHE_DIR: string = "./cache"; - public static generateCacheKey(url: string): string { - return createHash("md5").update(url).digest("hex"); + public static generateCacheKey(name: string): string { + return createHash("md5").update(name).digest("hex"); } - public static getCachePath(url: string): string { - const cacheKey = this.generateCacheKey(url); - return join(CacheService.CACHE_DIR, cacheKey); + public static getCachePath(name: string, extension: string = ""): string { + const cacheKey = this.generateCacheKey(name); + return join(CacheService.CACHE_DIR, `${cacheKey}${extension}`); } - public static isCached(url: string): boolean { - const cacheKey = CacheService.generateCacheKey(url); + public static isCached(name: string, extension: string = ""): boolean { + const cacheKey = CacheService.generateCacheKey(name); - const cachedPath = join(CacheService.CACHE_DIR, cacheKey); - return fs.pathExistsSync(cachedPath); + const cachedPath = join(CacheService.CACHE_DIR, `${cacheKey}${extension}`); + return existsSync(cachedPath); } } -fs.ensureDirSync(CacheService.CACHE_DIR); +if (!existsSync(CacheService.CACHE_DIR)) { + mkdirSync(CacheService.CACHE_DIR); +} export default CacheService; diff --git a/src/services/DMN.ts b/src/services/DMN.ts new file mode 100644 index 0000000..18830dc --- /dev/null +++ b/src/services/DMN.ts @@ -0,0 +1,10 @@ +import { Dataset } from "./dataset"; +import { Data } from "./data"; + +class DMN { + readonly xmlPath: string; + constructor(xmlPath: string, dataset: Dataset) { + this.xmlPath = xmlPath; + } + public parseXml() {} +} diff --git a/src/services/FileService.ts b/src/services/FileService.ts index bbbe8f4..052bf20 100644 --- a/src/services/FileService.ts +++ b/src/services/FileService.ts @@ -1,43 +1,37 @@ -import { extname } from "node:path"; -import { ArchiveExtractorFactory, ArchiveType } from "./archive_extractor"; +import { Readable } from "node:stream"; +import axios from "axios"; +import * as fs from "node:fs"; +import { WriteStream } from "node:fs"; class FileService { - public static async downloadAndExtract( - url: string, - output: string - ): Promise { - const fileType: string = FileService.getFileExtension(url); - const archiveExtractor = ArchiveExtractorFactory.getExtractor( - fileType as ArchiveType - ); - - console.log(`Downloading ${url}`); - const stream = await FileService.getFileStream(url); - return archiveExtractor.extract(stream, output); - } - - private static async getFileStream(url: string): Promise { - const response = await fetch(url, { + /** + * Get the compressed file stream from a given url + * @param url - The url of the file + * @return Promise - The compressed file stream + */ + public static async getFileStream(url: string): Promise { + return axios({ method: "GET", - headers: { - "Content-Type": "application/octet-stream", - }, - }); - - if (!response.ok) { - throw new Error(`Failed to download file: ${response.statusText}`); - } - - if (!response.body) { - throw new Error("Response body is not a readable stream"); - } - - return response.body; + url: url, + responseType: "stream", + }).then((response) => response.data); } - private static getFileExtension(url: string): string { - return extname(url).toLowerCase(); + /** + * Create a write stream to a file + * @param path - The path of the file + */ + public static createWriteStream(path: string): WriteStream { + return fs.createWriteStream(path); + } + + /** + * Delete a file from the file system + * @param cachePath - The path of the file to delete + */ + public static deleteFile(cachePath: string): void { + fs.unlinkSync(cachePath); } } diff --git a/src/services/archive/Archive.ts b/src/services/archive/Archive.ts new file mode 100644 index 0000000..33b3708 --- /dev/null +++ b/src/services/archive/Archive.ts @@ -0,0 +1,7 @@ +import { Duplex } from "node:stream"; + +interface Archive { + extract(source: string): Duplex; +} + +export default Archive; diff --git a/src/services/archive/ArchiveFactory.ts b/src/services/archive/ArchiveFactory.ts new file mode 100644 index 0000000..336a129 --- /dev/null +++ b/src/services/archive/ArchiveFactory.ts @@ -0,0 +1,12 @@ +import { Archive, ZipArchive, ArchiveType, GzipArchive } from "./"; + +class ArchiveFactory { + static getArchive(archiveType: ArchiveType): Archive { + if (archiveType === ArchiveType.ZIP) return ZipArchive.instance; + if ([ArchiveType.GZIP, ArchiveType.GZ].includes(archiveType)) return GzipArchive.instance; + + throw new Error("Unsupported archive type"); + } +} + +export default ArchiveFactory; diff --git a/src/services/archive/ArchiveType.ts b/src/services/archive/ArchiveType.ts new file mode 100644 index 0000000..843b5bb --- /dev/null +++ b/src/services/archive/ArchiveType.ts @@ -0,0 +1,7 @@ +enum ArchiveType { + ZIP = ".zip", + GZIP = ".gzip", + GZ = ".gz", +} + +export default ArchiveType; diff --git a/src/services/archive/GzipArchive.ts b/src/services/archive/GzipArchive.ts new file mode 100644 index 0000000..2c56bbe --- /dev/null +++ b/src/services/archive/GzipArchive.ts @@ -0,0 +1,12 @@ +import { createGunzip } from "node:zlib"; +import { Duplex } from "node:stream"; +import { Archive } from "./"; + +class GzipArchive implements Archive { + public static instance: Archive = new GzipArchive(); + + public extract(source: string): Duplex { + return createGunzip(); + } +} +export default GzipArchive; diff --git a/src/services/archive/ZipArchive.ts b/src/services/archive/ZipArchive.ts new file mode 100644 index 0000000..05d3f97 --- /dev/null +++ b/src/services/archive/ZipArchive.ts @@ -0,0 +1,15 @@ +import { Archive } from "./"; +import { Duplex } from "node:stream"; +import { ParseOne } from "unzipper"; + +class ZipArchive implements Archive { + public static instance: Archive = new ZipArchive(); + + public extract(source: string): Duplex { + return ParseOne(new RegExp(source), { + forceStream: true, + }); + } +} + +export default ZipArchive; diff --git a/src/services/archive/index.ts b/src/services/archive/index.ts new file mode 100644 index 0000000..7e1491b --- /dev/null +++ b/src/services/archive/index.ts @@ -0,0 +1,7 @@ +export { default as ArchiveType } from "./ArchiveType"; + +export { default as ArchiveFactory } from "./ArchiveFactory"; +export { default as Archive } from "./Archive"; + +export { default as ZipArchive } from "./ZipArchive"; +export { default as GzipArchive } from "./GzipArchive"; diff --git a/src/services/archive_extractor/ArchiveExtractor.ts b/src/services/archive_extractor/ArchiveExtractor.ts deleted file mode 100644 index f5c22e1..0000000 --- a/src/services/archive_extractor/ArchiveExtractor.ts +++ /dev/null @@ -1,5 +0,0 @@ -interface ArchiveExtractor { - extract(stream: ReadableStream, destinationPath: string): Promise; -} - -export default ArchiveExtractor; diff --git a/src/services/archive_extractor/ArchiveExtractorFactory.ts b/src/services/archive_extractor/ArchiveExtractorFactory.ts deleted file mode 100644 index 896e0e5..0000000 --- a/src/services/archive_extractor/ArchiveExtractorFactory.ts +++ /dev/null @@ -1,16 +0,0 @@ -import ArchiveExtractor from "./ArchiveExtractor"; -import ZipExtractor from "./ZipExtractor"; - -enum ArchiveType { - ZIP = ".zip", -} - -class ArchiveExtractorFactory { - static getExtractor(archiveType: ArchiveType): ArchiveExtractor { - if (archiveType === ArchiveType.ZIP) return ZipExtractor.instance; - throw new Error("Unsupported archive type"); - } -} - -export default ArchiveExtractorFactory; -export { ArchiveType }; diff --git a/src/services/archive_extractor/ZipExtractor.ts b/src/services/archive_extractor/ZipExtractor.ts deleted file mode 100644 index d67e433..0000000 --- a/src/services/archive_extractor/ZipExtractor.ts +++ /dev/null @@ -1,18 +0,0 @@ -import ArchiveExtractor from "./ArchiveExtractor"; -import unzipper from "unzipper"; -import { pipeline } from "node:stream"; -import { promisify } from "node:util"; - -class ZipExtractor implements ArchiveExtractor { - public static instance: ArchiveExtractor = new ZipExtractor(); - - async extract( - stream: ReadableStream, - destinationPath: string - ): Promise { - const streamPipeline = promisify(pipeline); - await streamPipeline(stream, unzipper.Extract({ path: destinationPath })); - } -} - -export default ZipExtractor; diff --git a/src/services/archive_extractor/index.ts b/src/services/archive_extractor/index.ts deleted file mode 100644 index c1e2c5d..0000000 --- a/src/services/archive_extractor/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export { default as ArchiveExtractor } from "./ArchiveExtractor"; -export { - default as ArchiveExtractorFactory, - ArchiveType, -} from "./ArchiveExtractorFactory"; -export { default as ZipExtractor } from "./ZipExtractor"; diff --git a/src/services/data/Data.ts b/src/services/data/Data.ts new file mode 100644 index 0000000..20d4aee --- /dev/null +++ b/src/services/data/Data.ts @@ -0,0 +1,3 @@ +interface Data {} + +export default Data; diff --git a/src/services/data/NudgerData.ts b/src/services/data/NudgerData.ts new file mode 100644 index 0000000..7fdfaf9 --- /dev/null +++ b/src/services/data/NudgerData.ts @@ -0,0 +1,28 @@ +import { Data } from "./"; + +type RawNudgerData = { + code: string; // "3260014791012", + brand: string; // "ALSATEK", + model: string; // "TL33171", + name: string; // "alsatek lg g3 coque protection aluminium rouge bumper tl33171", + last_updated: string; // "1562430134146", + gs1_country: string; // "FR", + offers_count: string; // "0", + min_price: string; // "", + min_price_compensation: string; // "", + currency: string; // "", + categories: string; // "ACCESSOIRES>COQUE SMARTPHONE", + url: string; // "" +}; + +class NudgerData implements Data { + barcode: string; + country: string; + + constructor(rawData: RawNudgerData) { + this.barcode = rawData.code; + this.country = rawData.gs1_country; + } +} + +export default NudgerData; diff --git a/src/services/data/index.ts b/src/services/data/index.ts new file mode 100644 index 0000000..f497037 --- /dev/null +++ b/src/services/data/index.ts @@ -0,0 +1,3 @@ +export {default as Data} from "./Data"; + +export {default as NudgerData} from "./NudgerData"; diff --git a/src/services/dataset/Dataset.ts b/src/services/dataset/Dataset.ts new file mode 100644 index 0000000..443936b --- /dev/null +++ b/src/services/dataset/Dataset.ts @@ -0,0 +1,76 @@ +import { pipeline } from "node:stream"; +import { promisify } from "node:util"; +import CacheService from "../CacheService"; +import FileService from "../FileService"; +import { ArchiveFactory, ArchiveType } from "../archive"; +import { ParserFactory } from "../parser"; +import { DatasetType } from "./"; + +/** + * Represents a dataset that can be loaded and queried + */ +class Dataset { + + readonly url: string; + readonly sourceFile: string; + readonly archiveType: ArchiveType; + readonly datasetType: DatasetType; + readonly cachePath: string; + + /** + * Create a new dataset instance + * @param url - The URL of the dataset + * @param sourceFile - The file name of the dataset in the archive + * @param archiveType - The type of the archive + * @param datasetType - The type of the dataset + */ + constructor( + url: string, + sourceFile: string, + archiveType: ArchiveType, + datasetType: DatasetType, + ) { + this.url = url; + this.sourceFile = sourceFile; + this.archiveType = archiveType; + this.datasetType = datasetType; + + this.cachePath = CacheService.getCachePath(this.url, ".json"); + } + + /** + * Load the dataset by downloading, extracting, parsing and saving it in cache + * @return Promise - A promise that resolves when the dataset is loaded + * @throws {Error} - If the dataset cannot be loaded + */ + public async load(): Promise { + if (CacheService.isCached(this.url, ".json")) { + console.log(`Already cached: ${this.url}`); + return; + } + + const archive = ArchiveFactory.getArchive(this.archiveType); + const parser = ParserFactory.getParser(this.datasetType); + + const pipelineAsync = promisify(pipeline); + + console.log(`Download: ${this.url}`); + await pipelineAsync( + await FileService.getFileStream(this.url), + archive.extract(this.sourceFile), + parser.parse(), + FileService.createWriteStream(this.cachePath), + ); + } + + /** + * Get a number of data entries from the dataset + * @param count - The number of data entries to get (default: 10) + */ + public get(count: number = 10): Data[] { + // TODO: Implement the get method + return []; + } +} + +export default Dataset; diff --git a/src/services/dataset/DatasetCollection.ts b/src/services/dataset/DatasetCollection.ts new file mode 100644 index 0000000..38223fc --- /dev/null +++ b/src/services/dataset/DatasetCollection.ts @@ -0,0 +1,20 @@ +import { Data, NudgerData } from "../data"; +import { ArchiveType } from "../archive"; +import { Dataset, DatasetType } from "./"; + +class DatasetCollection { + static datasets: Dataset[] = [ + new Dataset( + "https://files.opendatarchives.fr/data.cquest.org/open4goods/gtin-open-data.zip", + "open4goods-full-gtin-dataset.csv", + ArchiveType.ZIP, + DatasetType.CSV, + ), + ]; + + public static loadAll(): Promise { + return Promise.all(this.datasets.map((dataset) => dataset.load())); + } +} + +export default DatasetCollection; diff --git a/src/services/dataset/DatasetType.ts b/src/services/dataset/DatasetType.ts new file mode 100644 index 0000000..1c8c19f --- /dev/null +++ b/src/services/dataset/DatasetType.ts @@ -0,0 +1,10 @@ +enum DatasetType { + CSV = ".csv", + // TSV = ".tsv", + // PARQUET = ".parquet", + // JSONL = ".jsonl", + // XML = ".xml", + // RDF = ".rdf", +} + +export default DatasetType; diff --git a/src/services/dataset/NudgerDatasetService.ts b/src/services/dataset/NudgerDatasetService.ts deleted file mode 100644 index 6898dc3..0000000 --- a/src/services/dataset/NudgerDatasetService.ts +++ /dev/null @@ -1,40 +0,0 @@ -import FileService from "../FileService"; -import CacheService from "../CacheService"; -import { extname, join } from "node:path"; -import { DatasetParserFactory, DatasetType } from "../dataset_parser"; - -class NudgerDatasetService { - private static URL: string = - "https://files.opendatarchives.fr/data.cquest.org/open4goods/gtin-open-data.zip"; - private static SOURCE_FILE: string = "open4goods-full-gtin-dataset.csv"; - private static CACHE_PATH: string = CacheService.getCachePath( - NudgerDatasetService.URL - ); - - public static loadDataset(): Promise { - if (CacheService.isCached(NudgerDatasetService.URL)) { - return Promise.resolve(); - } - - return FileService.downloadAndExtract( - NudgerDatasetService.URL, - NudgerDatasetService.CACHE_PATH - ); - } - - public static parse() { - const extension = extname(NudgerDatasetService.SOURCE_FILE).toLowerCase(); - - const parser = DatasetParserFactory.getParser(extension as DatasetType); - return parser.parse(NudgerDatasetService.getSourcePath()); - } - - public static getSourcePath(): string { - return join( - NudgerDatasetService.CACHE_PATH, - NudgerDatasetService.SOURCE_FILE - ); - } -} - -export default NudgerDatasetService; diff --git a/src/services/dataset/index.ts b/src/services/dataset/index.ts new file mode 100644 index 0000000..1c35651 --- /dev/null +++ b/src/services/dataset/index.ts @@ -0,0 +1,3 @@ +export {default as DatasetCollection} from "./DatasetCollection"; +export {default as DatasetType} from "./DatasetType"; +export {default as Dataset} from "./Dataset"; diff --git a/src/services/dataset_parser/CsvParser.ts b/src/services/dataset_parser/CsvParser.ts deleted file mode 100644 index cce081a..0000000 --- a/src/services/dataset_parser/CsvParser.ts +++ /dev/null @@ -1,17 +0,0 @@ -import DatasetParser from "./DatasetParser"; -import * as fs from "node:fs"; -import Papa from "papaparse"; - -class CsvParser implements DatasetParser { - public static instance: CsvParser = new CsvParser(); - - async parse(filePath: string) { - const stream = fs.createReadStream(filePath); - Papa.parse(stream, { - worker: true, - step: (res) => console.log("Row:", res.data), - }); - } -} - -export default CsvParser; diff --git a/src/services/dataset_parser/DatasetParser.ts b/src/services/dataset_parser/DatasetParser.ts deleted file mode 100644 index c8d5c25..0000000 --- a/src/services/dataset_parser/DatasetParser.ts +++ /dev/null @@ -1,5 +0,0 @@ -interface DatasetParser { - parse(filePath: string): unknown; -} - -export default DatasetParser; diff --git a/src/services/dataset_parser/DatasetParserFactory.ts b/src/services/dataset_parser/DatasetParserFactory.ts deleted file mode 100644 index e5f47a1..0000000 --- a/src/services/dataset_parser/DatasetParserFactory.ts +++ /dev/null @@ -1,16 +0,0 @@ -import DatasetParser from "./DatasetParser"; -import CsvParser from "./CsvParser"; - -enum DatasetType { - CSV = ".csv", -} - -class DatasetParserFactory { - static getParser(fileType: DatasetType): DatasetParser { - if (fileType === DatasetType.CSV) return CsvParser.instance; - throw new Error("Unsupported file type"); - } -} - -export default DatasetParserFactory; -export { DatasetType }; diff --git a/src/services/dataset_parser/index.ts b/src/services/dataset_parser/index.ts deleted file mode 100644 index d73350e..0000000 --- a/src/services/dataset_parser/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export { default as CsvParser } from "./CsvParser"; -export { default as DatasetParser } from "./DatasetParser"; -export { - default as DatasetParserFactory, - DatasetType, -} from "./DatasetParserFactory"; diff --git a/src/services/parser/CsvParser.ts b/src/services/parser/CsvParser.ts new file mode 100644 index 0000000..768e06f --- /dev/null +++ b/src/services/parser/CsvParser.ts @@ -0,0 +1,13 @@ +import { Parser } from "./"; +import { Duplex } from "node:stream"; +import csv from "csvtojson"; + +class CsvParser implements Parser { + public static instance: CsvParser = new CsvParser(); + + public parse(): Duplex { + return csv(); + } +} + +export default CsvParser; diff --git a/src/services/parser/Parser.ts b/src/services/parser/Parser.ts new file mode 100644 index 0000000..2f13169 --- /dev/null +++ b/src/services/parser/Parser.ts @@ -0,0 +1,10 @@ +import { Duplex } from "node:stream"; + +interface Parser { + /** + * Parse the content of the stream into JSON objects + */ + parse(): Duplex; +} + +export default Parser; diff --git a/src/services/parser/ParserFactory.ts b/src/services/parser/ParserFactory.ts new file mode 100644 index 0000000..aff7853 --- /dev/null +++ b/src/services/parser/ParserFactory.ts @@ -0,0 +1,16 @@ +import { Parser, CsvParser } from "./"; +import { DatasetType } from "../dataset"; + +class ParserFactory { + /** + * Get the parser corresponding to the dataset type + * @param fileType The type of the dataset + * @returns The parser corresponding to the dataset type + */ + static getParser(fileType: DatasetType): Parser { + if (fileType === DatasetType.CSV) return CsvParser.instance; + throw new Error("Unsupported file type"); + } +} + +export default ParserFactory; diff --git a/src/services/parser/index.ts b/src/services/parser/index.ts new file mode 100644 index 0000000..12ea722 --- /dev/null +++ b/src/services/parser/index.ts @@ -0,0 +1,4 @@ +export { default as ParserFactory } from "./ParserFactory"; +export { default as Parser } from "./Parser"; + +export { default as CsvParser } from "./CsvParser";