diff --git a/docker-compose.yaml b/docker-compose.yaml index 4fcfe47..1527f2d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,14 +1,13 @@ -version: '3.1' +version: "3.1" services: - mariadb: image: mariadb:latest - restart: 'always' - volumes: - - ./mariaDB/init:/docker-entrypoint-initdb.d + restart: "always" + # volumes: + # - ./mariaDB/init:/docker-entrypoint-initdb.d environment: MYSQL_ROOT_PASSWORD: root - MYSQL_DATABASE: db_prod + MYSQL_DATABASE: db ports: - - "3306:3306" \ No newline at end of file + - "3306:3306" diff --git a/src/routes/randomize.ts b/src/routes/randomize.ts index 4e42fe9..148fe41 100644 --- a/src/routes/randomize.ts +++ b/src/routes/randomize.ts @@ -1,7 +1,7 @@ import { Router, Request, Response } from "express"; import { DatasetCollection } from "../services/dataset"; import { DMN } from "../services/dmn/DMN"; -import { DMN_Definitions } from "../services/dmn/interfaces/DMN_Definitions"; +import { Definitions } from "../services/dmn/interfaces/"; import { Data } from "../services/data"; const router = Router(); @@ -22,7 +22,7 @@ router.post("/randomize/:id", async (req: Request, res: Response) => { ); if (!dataset) return res.status(404).json({ status: "NOT_FOUND" }); - const dmn: DMN_Definitions = await DMN.parse(req.body); + const dmn: Definitions = await DMN.parse(req.body); const schema = DMN.getSchema(dmn); const data: Data[] = await dataset.get(size, schema); diff --git a/src/services/archive/ArchiveFactory.ts b/src/services/archive/ArchiveFactory.ts index d46261d..1b061d3 100644 --- a/src/services/archive/ArchiveFactory.ts +++ b/src/services/archive/ArchiveFactory.ts @@ -3,8 +3,7 @@ import { Archive, ZipArchive, ArchiveType, GzipArchive } from "./"; class ArchiveFactory { static getArchive(archiveType: ArchiveType): Archive { if (archiveType === ArchiveType.ZIP) return ZipArchive.instance; - // if ([ArchiveType.GZIP, ArchiveType.GZ].includes(archiveType)) - // return GzipArchive.instance; + if (archiveType === ArchiveType.GZIP) return GzipArchive.instance; throw new Error("Unsupported archive type"); } diff --git a/src/services/archive/ArchiveType.ts b/src/services/archive/ArchiveType.ts index 843b5bb..ee9f365 100644 --- a/src/services/archive/ArchiveType.ts +++ b/src/services/archive/ArchiveType.ts @@ -1,7 +1,6 @@ enum ArchiveType { ZIP = ".zip", GZIP = ".gzip", - GZ = ".gz", } export default ArchiveType; diff --git a/src/services/archive/GzipArchive.ts b/src/services/archive/GzipArchive.ts index ce8ecdc..cae9caf 100644 --- a/src/services/archive/GzipArchive.ts +++ b/src/services/archive/GzipArchive.ts @@ -9,4 +9,5 @@ class GzipArchive implements Archive { return createGunzip(); } } + export default GzipArchive; diff --git a/src/services/data/OpenFoodFactsData.ts b/src/services/data/OpenFoodFactsData.ts new file mode 100644 index 0000000..1562d14 --- /dev/null +++ b/src/services/data/OpenFoodFactsData.ts @@ -0,0 +1,18 @@ +import { Data } from "./"; + +type RawOpenFoodFactsData = { + code: string; + countries_en: string; +}; + +class OpenFoodFactsData implements Data { + input: string[] = []; + output: string[] = []; + + constructor({ code, countries_en }: RawOpenFoodFactsData) { + this.input = [code]; + this.output = [countries_en]; + } +} + +export default OpenFoodFactsData; diff --git a/src/services/data/index.ts b/src/services/data/index.ts index ace1f9d..975abd8 100644 --- a/src/services/data/index.ts +++ b/src/services/data/index.ts @@ -1,3 +1,4 @@ export { default as Data, DataConstructor } from "./Data"; export { default as NudgerData } from "./NudgerData"; +export { default as OpenFoodFactsData } from "./OpenFoodFactsData"; diff --git a/src/services/dataset/Dataset.ts b/src/services/dataset/Dataset.ts index 836a15e..8b7ab45 100644 --- a/src/services/dataset/Dataset.ts +++ b/src/services/dataset/Dataset.ts @@ -75,8 +75,6 @@ class Dataset { const pipelineAsync = promisify(pipeline); - const self = this; - console.log(`Download: ${this.source}`); await pipelineAsync( await FileService.getFileStream(this.source), diff --git a/src/services/dataset/DatasetCollection.ts b/src/services/dataset/DatasetCollection.ts index 49da200..df94cdc 100644 --- a/src/services/dataset/DatasetCollection.ts +++ b/src/services/dataset/DatasetCollection.ts @@ -1,4 +1,4 @@ -import { NudgerData } from "../data"; +import { NudgerData, OpenFoodFactsData } from "../data"; import { ArchiveType } from "../archive"; import { Dataset, DatasetType } from "./"; @@ -13,6 +13,15 @@ class DatasetCollection { archiveType: ArchiveType.ZIP, datasetType: DatasetType.CSV, }), + new Dataset({ + id: "openfoodfacts", + source: + "https://static.openfoodfacts.org/data/en.openfoodfacts.org.products.csv.gz", + file: "en.openfoodfacts.org.products.csv", + dataType: OpenFoodFactsData, + archiveType: ArchiveType.GZIP, + datasetType: DatasetType.CSV, + }), ]; public static loadAll(): Promise { diff --git a/src/services/parser/CsvParser.ts b/src/services/parser/CsvParser.ts index 768e06f..073e326 100644 --- a/src/services/parser/CsvParser.ts +++ b/src/services/parser/CsvParser.ts @@ -6,7 +6,9 @@ class CsvParser implements Parser { public static instance: CsvParser = new CsvParser(); public parse(): Duplex { - return csv(); + return csv({ + delimiter: "auto", + }); } }