draft: Add OpenFoodFacts Dataset

/!\ Warning: Stream doesn't end

docker-compose.yaml

@@ -1,14 +1,13 @@
-version: '3.1'
+version: "3.1"
 
 services:
-
   mariadb:
     image: mariadb:latest
-    restart: 'always'
-    volumes:
-      - ./mariaDB/init:/docker-entrypoint-initdb.d
+    restart: "always"
+    #    volumes:
+    #      - ./mariaDB/init:/docker-entrypoint-initdb.d
     environment:
       MYSQL_ROOT_PASSWORD: root
-      MYSQL_DATABASE: db_prod
+      MYSQL_DATABASE: db
     ports:
       - "3306:3306"

src/routes/randomize.ts

@@ -1,7 +1,7 @@
 import { Router, Request, Response } from "express";
 import { DatasetCollection } from "../services/dataset";
 import { DMN } from "../services/dmn/DMN";
-import { DMN_Definitions } from "../services/dmn/interfaces/DMN_Definitions";
+import { Definitions } from "../services/dmn/interfaces/";
 import { Data } from "../services/data";
 
 const router = Router();
@@ -22,7 +22,7 @@ router.post("/randomize/:id", async (req: Request, res: Response) => {
   );
   if (!dataset) return res.status(404).json({ status: "NOT_FOUND" });
 
-  const dmn: DMN_Definitions = await DMN.parse(req.body);
+  const dmn: Definitions = await DMN.parse(req.body);
   const schema = DMN.getSchema(dmn);
 
   const data: Data[] = await dataset.get(size, schema);

src/services/archive/ArchiveFactory.ts

@@ -3,8 +3,7 @@ import { Archive, ZipArchive, ArchiveType, GzipArchive } from "./";
 class ArchiveFactory {
   static getArchive(archiveType: ArchiveType): Archive {
     if (archiveType === ArchiveType.ZIP) return ZipArchive.instance;
-    // if ([ArchiveType.GZIP, ArchiveType.GZ].includes(archiveType))
-    // return GzipArchive.instance;
+    if (archiveType === ArchiveType.GZIP) return GzipArchive.instance;
 
     throw new Error("Unsupported archive type");
   }

src/services/archive/ArchiveType.ts

@@ -1,7 +1,6 @@
 enum ArchiveType {
   ZIP = ".zip",
   GZIP = ".gzip",
-  GZ = ".gz",
 }
 
 export default ArchiveType;

src/services/archive/GzipArchive.ts

@@ -9,4 +9,5 @@ class GzipArchive implements Archive {
     return createGunzip();
   }
 }
+
 export default GzipArchive;

src/services/data/OpenFoodFactsData.ts

@@ -0,0 +1,18 @@
+import { Data } from "./";
+
+type RawOpenFoodFactsData = {
+  code: string;
+  countries_en: string;
+};
+
+class OpenFoodFactsData implements Data {
+  input: string[] = [];
+  output: string[] = [];
+
+  constructor({ code, countries_en }: RawOpenFoodFactsData) {
+    this.input = [code];
+    this.output = [countries_en];
+  }
+}
+
+export default OpenFoodFactsData;

src/services/data/index.ts

@@ -1,3 +1,4 @@
 export { default as Data, DataConstructor } from "./Data";
 
 export { default as NudgerData } from "./NudgerData";
+export { default as OpenFoodFactsData } from "./OpenFoodFactsData";

src/services/dataset/Dataset.ts

@@ -75,8 +75,6 @@ class Dataset {
 
     const pipelineAsync = promisify(pipeline);
 
-    const self = this;
-
     console.log(`Download: ${this.source}`);
     await pipelineAsync(
       await FileService.getFileStream(this.source),

src/services/dataset/DatasetCollection.ts

@@ -1,4 +1,4 @@
-import { NudgerData } from "../data";
+import { NudgerData, OpenFoodFactsData } from "../data";
 import { ArchiveType } from "../archive";
 import { Dataset, DatasetType } from "./";
 
@@ -13,6 +13,15 @@ class DatasetCollection {
       archiveType: ArchiveType.ZIP,
       datasetType: DatasetType.CSV,
     }),
+    new Dataset({
+      id: "openfoodfacts",
+      source:
+        "https://static.openfoodfacts.org/data/en.openfoodfacts.org.products.csv.gz",
+      file: "en.openfoodfacts.org.products.csv",
+      dataType: OpenFoodFactsData,
+      archiveType: ArchiveType.GZIP,
+      datasetType: DatasetType.CSV,
+    }),
   ];
 
   public static loadAll(): Promise<void[]> {

src/services/parser/CsvParser.ts

@@ -6,7 +6,9 @@ class CsvParser implements Parser {
   public static instance: CsvParser = new CsvParser();
 
   public parse(): Duplex {
-    return csv();
+    return csv({
+      delimiter: "auto",
+    });
   }
 }