feat: Stream download, extract and parse datasets

src/services/dataset/Dataset.ts

@@ -0,0 +1,76 @@
+import { pipeline } from "node:stream";
+import { promisify } from "node:util";
+import CacheService from "../CacheService";
+import FileService from "../FileService";
+import { ArchiveFactory, ArchiveType } from "../archive";
+import { ParserFactory } from "../parser";
+import { DatasetType } from "./";
+
+/**
+ * Represents a dataset that can be loaded and queried
+ */
+class Dataset<Data> {
+
+  readonly url: string;
+  readonly sourceFile: string;
+  readonly archiveType: ArchiveType;
+  readonly datasetType: DatasetType;
+  readonly cachePath: string;
+
+  /**
+   * Create a new dataset instance
+   * @param url - The URL of the dataset
+   * @param sourceFile - The file name of the dataset in the archive
+   * @param archiveType - The type of the archive
+   * @param datasetType - The type of the dataset
+   */
+  constructor(
+    url: string,
+    sourceFile: string,
+    archiveType: ArchiveType,
+    datasetType: DatasetType,
+  ) {
+    this.url = url;
+    this.sourceFile = sourceFile;
+    this.archiveType = archiveType;
+    this.datasetType = datasetType;
+
+    this.cachePath = CacheService.getCachePath(this.url, ".json");
+  }
+
+  /**
+   * Load the dataset by downloading, extracting, parsing and saving it in cache
+   * @return Promise<void> - A promise that resolves when the dataset is loaded
+   * @throws {Error} - If the dataset cannot be loaded
+   */
+  public async load(): Promise<void> {
+    if (CacheService.isCached(this.url, ".json")) {
+      console.log(`Already cached: ${this.url}`);
+      return;
+    }
+
+    const archive = ArchiveFactory.getArchive(this.archiveType);
+    const parser = ParserFactory.getParser(this.datasetType);
+
+    const pipelineAsync = promisify(pipeline);
+
+    console.log(`Download: ${this.url}`);
+    await pipelineAsync(
+      await FileService.getFileStream(this.url),
+      archive.extract(this.sourceFile),
+      parser.parse(),
+      FileService.createWriteStream(this.cachePath),
+    );
+  }
+
+  /**
+   * Get a number of data entries from the dataset
+   * @param count - The number of data entries to get (default: 10)
+   */
+  public get(count: number = 10): Data[] {
+    // TODO: Implement the get method
+    return [];
+  }
+}
+
+export default Dataset;

src/services/dataset/DatasetCollection.ts

@@ -0,0 +1,20 @@
+import { Data, NudgerData } from "../data";
+import { ArchiveType } from "../archive";
+import { Dataset, DatasetType } from "./";
+
+class DatasetCollection {
+  static datasets: Dataset<Data>[] = [
+    new Dataset<NudgerData>(
+      "https://files.opendatarchives.fr/data.cquest.org/open4goods/gtin-open-data.zip",
+      "open4goods-full-gtin-dataset.csv",
+      ArchiveType.ZIP,
+      DatasetType.CSV,
+    ),
+  ];
+
+  public static loadAll(): Promise<void[]> {
+    return Promise.all(this.datasets.map((dataset) => dataset.load()));
+  }
+}
+
+export default DatasetCollection;

src/services/dataset/DatasetType.ts

@@ -0,0 +1,10 @@
+enum DatasetType {
+  CSV = ".csv",
+  // TSV = ".tsv",
+  // PARQUET = ".parquet",
+  // JSONL = ".jsonl",
+  // XML = ".xml",
+  // RDF = ".rdf",
+}
+
+export default DatasetType;

src/services/dataset/NudgerDatasetService.ts

@@ -1,40 +0,0 @@
-import FileService from "../FileService";
-import CacheService from "../CacheService";
-import { extname, join } from "node:path";
-import { DatasetParserFactory, DatasetType } from "../dataset_parser";
-
-class NudgerDatasetService {
-  private static URL: string =
-    "https://files.opendatarchives.fr/data.cquest.org/open4goods/gtin-open-data.zip";
-  private static SOURCE_FILE: string = "open4goods-full-gtin-dataset.csv";
-  private static CACHE_PATH: string = CacheService.getCachePath(
-    NudgerDatasetService.URL
-  );
-
-  public static loadDataset(): Promise<void> {
-    if (CacheService.isCached(NudgerDatasetService.URL)) {
-      return Promise.resolve();
-    }
-
-    return FileService.downloadAndExtract(
-      NudgerDatasetService.URL,
-      NudgerDatasetService.CACHE_PATH
-    );
-  }
-
-  public static parse() {
-    const extension = extname(NudgerDatasetService.SOURCE_FILE).toLowerCase();
-
-    const parser = DatasetParserFactory.getParser(extension as DatasetType);
-    return parser.parse(NudgerDatasetService.getSourcePath());
-  }
-
-  public static getSourcePath(): string {
-    return join(
-      NudgerDatasetService.CACHE_PATH,
-      NudgerDatasetService.SOURCE_FILE
-    );
-  }
-}
-
-export default NudgerDatasetService;

src/services/dataset/index.ts

@@ -0,0 +1,3 @@
+export {default as DatasetCollection} from "./DatasetCollection";
+export {default as DatasetType} from "./DatasetType";
+export {default as Dataset} from "./Dataset";