feat!: Download and extract from stream

src/Server.ts

@@ -0,0 +1,21 @@
+import express from "express";
+import routes from "./routes";
+import { createServer } from "node:http";
+import { logger } from "./middlewares";
+
+export default class Server {
+  private readonly app: express.Application;
+  private static PORT: number = 4321;
+
+  constructor() {
+    this.app = express();
+    this.app.use(logger, routes);
+  }
+
+  public start() {
+    const server = createServer(this.app);
+    server.listen(Server.PORT, () => {
+      console.info(`Server is running on http://localhost:${Server.PORT}`);
+    });
+  }
+}

src/index.ts

@@ -1,17 +1,9 @@
-import express from "express";
 import dotenv from "dotenv";
-import { createServer } from "node:http";
-import { logger } from "./middlewares";
-import routes from "./routes";
+import Server from "./Server";
+import NudgerDatasetService from "./services/dataset/NudgerDatasetService";
 
 dotenv.config();
 
-const app = express();
-app.use(logger, routes);
-const server = createServer(app);
-
-server.listen(process.env.PORT || 8080, () => {
-  console.info(
-    `Server is running on http://localhost:${process.env.PORT || 8080}`
-  );
-});
+Promise.all([NudgerDatasetService.loadDataset()])
+  .then(() => new Server().start())
+  .catch(console.error);

src/routes/data/nudger.ts

@@ -1,20 +1,9 @@
 import { Router, Request, Response } from "express";
-import fileService from "../../services/fileService";
 
 const router = Router();
 
 router.get("/data/nudger", (req: Request, res: Response) => {
-  fileService
-    .downloadAndExtract("https://nudger.fr/opendata/gtin-open-data.zip")
-    .then(() => {
-      res.status(200).json({
-        status: "SUCCESS",
-        message: "Data nudger downloaded and extracted",
-      });
-    })
-    .catch((error) => {
-      res.status(500).json({ status: "ERROR", message: error.message });
-    });
+  res.status(501).send("Not yet implemented");
 });
 
 export default router;

src/routes/data/openfoodfacts.ts

@@ -1,22 +1,9 @@
 import { Router, Request, Response } from "express";
-import fileService from "../../services/fileService";
 
 const router = Router();
 
 router.get("/data/openfoodfacts", (req: Request, res: Response) => {
-  fileService
-    .downloadAndExtract(
-      "https://static.openfoodfacts.org/data/en.openfoodfacts.org.products.csv.gz"
-    )
-    .then(() => {
-      res.status(200).json({
-        status: "SUCCESS",
-        message: "Data openfoodfacts downloaded and extracted",
-      });
-    })
-    .catch((error) => {
-      res.status(500).json({ status: "ERROR", message: error.message });
-    });
+  res.status(501).send("Not yet implemented");
 });
 
 export default router;

src/services/CacheService.ts

@@ -0,0 +1,27 @@
+import { createHash } from "node:crypto";
+import { join } from "node:path";
+import * as fs from "fs-extra";
+
+class CacheService {
+  public static readonly CACHE_DIR: string = "./cache";
+
+  public static generateCacheKey(url: string): string {
+    return createHash("md5").update(url).digest("hex");
+  }
+
+  public static getCachePath(url: string): string {
+    const cacheKey = this.generateCacheKey(url);
+    return join(CacheService.CACHE_DIR, cacheKey);
+  }
+
+  public static isCached(url: string): boolean {
+    const cacheKey = CacheService.generateCacheKey(url);
+
+    const cachedPath = join(CacheService.CACHE_DIR, cacheKey);
+    return fs.pathExistsSync(cachedPath);
+  }
+}
+
+fs.ensureDirSync(CacheService.CACHE_DIR);
+
+export default CacheService;

src/services/FileService.ts

@@ -0,0 +1,44 @@
+import { extname } from "node:path";
+import { ArchiveExtractorFactory, ArchiveType } from "./archive_extractor";
+
+class FileService {
+  public static async downloadAndExtract(
+    url: string,
+    output: string
+  ): Promise<void> {
+    const fileType: string = FileService.getFileExtension(url);
+
+    const archiveExtractor = ArchiveExtractorFactory.getExtractor(
+      fileType as ArchiveType
+    );
+
+    console.log(`Downloading ${url}`);
+    const stream = await FileService.getFileStream(url);
+    return archiveExtractor.extract(stream, output);
+  }
+
+  private static async getFileStream(url: string): Promise<ReadableStream> {
+    const response = await fetch(url, {
+      method: "GET",
+      headers: {
+        "Content-Type": "application/octet-stream",
+      },
+    });
+
+    if (!response.ok) {
+      throw new Error(`Failed to download file: ${response.statusText}`);
+    }
+
+    if (!response.body) {
+      throw new Error("Response body is not a readable stream");
+    }
+
+    return response.body;
+  }
+
+  private static getFileExtension(url: string): string {
+    return extname(url).toLowerCase();
+  }
+}
+
+export default FileService;

src/services/archive_extractor/ArchiveExtractor.ts

@@ -0,0 +1,5 @@
+interface ArchiveExtractor {
+  extract(stream: ReadableStream, destinationPath: string): Promise<void>;
+}
+
+export default ArchiveExtractor;

src/services/archive_extractor/ArchiveExtractorFactory.ts

@@ -0,0 +1,16 @@
+import ArchiveExtractor from "./ArchiveExtractor";
+import ZipExtractor from "./ZipExtractor";
+
+enum ArchiveType {
+  ZIP = ".zip",
+}
+
+class ArchiveExtractorFactory {
+  static getExtractor(archiveType: ArchiveType): ArchiveExtractor {
+    if (archiveType === ArchiveType.ZIP) return ZipExtractor.instance;
+    throw new Error("Unsupported archive type");
+  }
+}
+
+export default ArchiveExtractorFactory;
+export { ArchiveType };

src/services/archive_extractor/ZipExtractor.ts

@@ -0,0 +1,18 @@
+import ArchiveExtractor from "./ArchiveExtractor";
+import unzipper from "unzipper";
+import { pipeline } from "node:stream";
+import { promisify } from "node:util";
+
+class ZipExtractor implements ArchiveExtractor {
+  public static instance: ArchiveExtractor = new ZipExtractor();
+
+  async extract(
+    stream: ReadableStream,
+    destinationPath: string
+  ): Promise<void> {
+    const streamPipeline = promisify(pipeline);
+    await streamPipeline(stream, unzipper.Extract({ path: destinationPath }));
+  }
+}
+
+export default ZipExtractor;

src/services/archive_extractor/index.ts

@@ -0,0 +1,6 @@
+export { default as ArchiveExtractor } from "./ArchiveExtractor";
+export {
+  default as ArchiveExtractorFactory,
+  ArchiveType,
+} from "./ArchiveExtractorFactory";
+export { default as ZipExtractor } from "./ZipExtractor";

src/services/dataset/NudgerDatasetService.ts

@@ -0,0 +1,40 @@
+import FileService from "../FileService";
+import CacheService from "../CacheService";
+import { extname, join } from "node:path";
+import { DatasetParserFactory, DatasetType } from "../dataset_parser";
+
+class NudgerDatasetService {
+  private static URL: string =
+    "https://files.opendatarchives.fr/data.cquest.org/open4goods/gtin-open-data.zip";
+  private static SOURCE_FILE: string = "open4goods-full-gtin-dataset.csv";
+  private static CACHE_PATH: string = CacheService.getCachePath(
+    NudgerDatasetService.URL
+  );
+
+  public static loadDataset(): Promise<void> {
+    if (CacheService.isCached(NudgerDatasetService.URL)) {
+      return Promise.resolve();
+    }
+
+    return FileService.downloadAndExtract(
+      NudgerDatasetService.URL,
+      NudgerDatasetService.CACHE_PATH
+    );
+  }
+
+  public static parse() {
+    const extension = extname(NudgerDatasetService.SOURCE_FILE).toLowerCase();
+
+    const parser = DatasetParserFactory.getParser(extension as DatasetType);
+    return parser.parse(NudgerDatasetService.getSourcePath());
+  }
+
+  public static getSourcePath(): string {
+    return join(
+      NudgerDatasetService.CACHE_PATH,
+      NudgerDatasetService.SOURCE_FILE
+    );
+  }
+}
+
+export default NudgerDatasetService;

src/services/dataset_parser/CsvParser.ts

@@ -0,0 +1,17 @@
+import DatasetParser from "./DatasetParser";
+import * as fs from "node:fs";
+import Papa from "papaparse";
+
+class CsvParser implements DatasetParser {
+  public static instance: CsvParser = new CsvParser();
+
+  async parse(filePath: string) {
+    const stream = fs.createReadStream(filePath);
+    Papa.parse(stream, {
+      worker: true,
+      step: (res) => console.log("Row:", res.data),
+    });
+  }
+}
+
+export default CsvParser;

src/services/dataset_parser/DatasetParser.ts

@@ -0,0 +1,5 @@
+interface DatasetParser {
+  parse(filePath: string): unknown;
+}
+
+export default DatasetParser;

src/services/dataset_parser/DatasetParserFactory.ts

@@ -0,0 +1,16 @@
+import DatasetParser from "./DatasetParser";
+import CsvParser from "./CsvParser";
+
+enum DatasetType {
+  CSV = ".csv",
+}
+
+class DatasetParserFactory {
+  static getParser(fileType: DatasetType): DatasetParser {
+    if (fileType === DatasetType.CSV) return CsvParser.instance;
+    throw new Error("Unsupported file type");
+  }
+}
+
+export default DatasetParserFactory;
+export { DatasetType };

src/services/dataset_parser/index.ts

@@ -0,0 +1,6 @@
+export { default as CsvParser } from "./CsvParser";
+export { default as DatasetParser } from "./DatasetParser";
+export {
+  default as DatasetParserFactory,
+  DatasetType,
+} from "./DatasetParserFactory";

src/services/fileService.ts

@@ -1,131 +0,0 @@
-import axios from "axios";
-import * as unzipper from "unzipper";
-import * as fs from "fs-extra";
-import * as zlib from "zlib";
-import { extname, join, basename } from "path";
-import crypto from "crypto"; // Utilisé pour générer des identifiants uniques basés sur l'URL
-
-type SupportedFormats = "zip" | "gz" | "gzip";
-
-class FileService {
-  private cacheDir: string;
-
-  constructor() {
-    this.cacheDir = "./cache";
-    fs.ensureDirSync(this.cacheDir);
-  }
-
-  /**
-   * Télécharger et extraire le fichier à partir de l'URL
-   * @param url URL du fichier à télécharger
-   */
-  async downloadAndExtract(url: string): Promise<void> {
-    try {
-      const fileType = this.getFileExtension(url);
-      if (!fileType) throw new Error("Unsupported file format");
-      if (this.isInCache(url)) return;
-
-      const response = await axios({
-        method: "GET",
-        url,
-        responseType: "stream",
-      });
-      console.log(`Downloading : ${url}`);
-
-      // Décompresser et sauvegarder dans le cache
-      const cacheKey = this.generateCacheKey(url);
-      const cachedPath = join(this.cacheDir, cacheKey);
-
-      fs.ensureDirSync(cachedPath);
-
-      if (fileType === "zip") await this.extractZip(response.data, cachedPath);
-      if (fileType === "gz" || fileType === "gzip")
-        await this.extractGzip(
-          response.data,
-          join(cachedPath, basename(url).replace(/\.(gz|gzip)$/, ""))
-        );
-
-      console.log(`Downloaded and extracted : ${basename(url)}`);
-    } catch (error) {
-      console.error(
-        "An error occurred while downloading and extracting the file",
-        error
-      );
-    }
-  }
-
-  /**
-   * Vérifier si le fichier est déjà en cache
-   * @param url URL du fichier
-   * @private
-   */
-  private isInCache(url: string): boolean {
-    const cacheKey = this.generateCacheKey(url);
-    const cachedPath = join(this.cacheDir, cacheKey);
-    return fs.pathExistsSync(cachedPath);
-  }
-
-  /**
-   * Extraire les fichiers ZIP et stocker dans le cache
-   * @param stream
-   * @param cachePath
-   * @private
-   */
-  private async extractZip(
-    stream: NodeJS.ReadableStream,
-    cachePath: string
-  ): Promise<void> {
-    return new Promise((resolve, reject) => {
-      stream
-        .pipe(unzipper.Extract({ path: cachePath }))
-        .on("close", resolve)
-        .on("error", reject);
-    });
-  }
-
-  /**
-   * Extraire les fichiers GZ et GZIP et stocker dans le cache
-   * @param stream Flux du fichier téléchargé
-   * @param cachePath Chemin où stocker le fichier décompressé
-   * @private
-   */
-  private async extractGzip(
-    stream: NodeJS.ReadableStream,
-    cachePath: string
-  ): Promise<void> {
-    return new Promise((resolve, reject) => {
-      // Ajouter une extension correcte (par exemple, si le fichier original est 'file.gz', le résultat sera 'file')
-      const decompressedFilePath = cachePath.replace(/\.gz$/, "");
-
-      const writeStream = fs.createWriteStream(decompressedFilePath);
-
-      // Pipeliner le flux du téléchargement et la décompression
-      stream
-        .pipe(zlib.createGunzip()) // Décompresser le flux
-        .pipe(writeStream) // Écrire le fichier décompressé
-        .on("finish", resolve)
-        .on("error", reject);
-    });
-  }
-
-  /**
-   * Obtenir l'extension du fichier à partir de l'URL
-   * @param url URL du fichier
-   */
-  private getFileExtension(url: string): SupportedFormats | null {
-    const extension = extname(url).toLowerCase();
-    if (extension === ".zip") return "zip";
-    if (extension === ".gz" || extension === ".gzip") return "gz";
-    return null;
-  }
-
-  /**
-   * Générer un identifiant unique pour le fichier basé sur l'URL
-   * @param url URL du fichier
-   */
-  private generateCacheKey(url: string): string {
-    return crypto.createHash("md5").update(url).digest("hex");
-  }
-}
-
-export default new FileService();