feat: Stream download, extract and parse datasets

This commit is contained in:
Lucàs
2024-09-28 16:19:43 +02:00
parent 5a00871319
commit ffc1ad3e84
32 changed files with 501 additions and 791 deletions
+20
View File
@@ -0,0 +1,20 @@
import { Data, NudgerData } from "../data";
import { ArchiveType } from "../archive";
import { Dataset, DatasetType } from "./";
class DatasetCollection {
static datasets: Dataset<Data>[] = [
new Dataset<NudgerData>(
"https://files.opendatarchives.fr/data.cquest.org/open4goods/gtin-open-data.zip",
"open4goods-full-gtin-dataset.csv",
ArchiveType.ZIP,
DatasetType.CSV,
),
];
public static loadAll(): Promise<void[]> {
return Promise.all(this.datasets.map((dataset) => dataset.load()));
}
}
export default DatasetCollection;