V_1.0.0

Dockerfile_ExerciceChecker

@@ -31,6 +31,9 @@ FROM docker:latest
 LABEL maintainer="Michaël Minelli <michael-jean.minelli@hesge.ch>"
 LABEL Description="Dojo exercice checker - Container generation for usage in exercice pipelines"
 
+RUN apk update
+RUN apk add git
+
 ARG BUILD_WORKDIR
 
 COPY --from=builder ${BUILD_WORKDIR}/bin/app /usr/local/bin/dojo_exercice_checker
\ No newline at end of file

ExerciceChecker/.idea/vcs.xml

@@ -2,5 +2,7 @@
 <project version="4">
   <component name="VcsDirectoryMappings">
     <mapping directory="$PROJECT_DIR$/.." vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/src/shared" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/src/sharedByClients" vcs="Git" />
   </component>
 </project>
\ No newline at end of file

ExerciceChecker/assets/docker-compose-override.yml

+volumes:
+      {{VOLUME_NAME}}:
+          driver_opts:
+              type  : none
+              device: {{MOUNT_PATH}}
+              o     : bind

ExerciceChecker/package.json

 {
     "name"           : "dojo_exercice_checker",
-    "version"        : "0.0.1",
+    "version"        : "1.0.0",
     "main"           : "dist/app.js",
     "bin"            : {
         "dirmanager": "./dist/app.js"
@@ -9,7 +9,8 @@
         "scripts": [],
         "assets" : [
             "node_modules/axios/dist/node/axios.cjs",
-            ".env"
+            ".env",
+            "assets/**/*"
         ],
         "targets": [
             "node18-linux-arm64",
@@ -22,15 +23,25 @@
         "test"     : "echo \"Error: no test specified\" && exit 1"
     },
     "dependencies"   : {
+        "ajv"              : "^8.12.0",
         "axios"            : "^1.4.0",
+        "boxen"            : "^5.1.2",
         "chalk"            : "^4.1.2",
+        "dockerode"        : "^3.3.5",
         "dotenv"           : "^16.3.1",
+        "fs-extra"         : "^11.1.1",
         "http-status-codes": "^2.2.0",
+        "json5"            : "^2.2.3",
         "ora"              : "^5.4.1",
+        "tar-stream"       : "^3.1.6",
         "winston"          : "^3.10.0"
     },
     "devDependencies": {
+        "@types/dockerode" : "^3.3.19",
+        "@types/fs-extra"  : "^11.0.1",
+        "@types/js-yaml"   : "^4.0.5",
         "@types/node"      : "^18.17.1",
+        "@types/tar-stream": "^2.2.2",
         "pkg"              : "^5.8.1",
         "ts-node"          : "^10.9.1",
         "typescript"       : "^5.1.6"

ExerciceChecker/src/app.ts

@@ -4,20 +4,173 @@ const path = require('node:path');
 require('dotenv').config({ path: path.join(__dirname, '../.env') });
 require('./shared/helpers/TypeScriptExtensions'); // ATTENTION : This line MUST be the second of this file
 
-import chalk       from 'chalk';
+import Styles               from './types/Styles';
+import Icon                 from './types/Icon';
+import boxen                from 'boxen';
+import RecursiveFilesStats  from './shared/helpers/recursiveFilesStats/RecursiveFilesStats';
+import Toolbox              from './shared/helpers/Toolbox';
+import ExerciceHelper       from './shared/helpers/ExerciceHelper';
+import ExerciceCheckerError from './types/ExerciceCheckerError';
+import { exec, spawn }      from 'child_process';
+import util                 from 'util';
+import fs                   from 'fs-extra';
 import HttpManager          from './managers/HttpManager';
+import DojoBackendManager   from './managers/DojoBackendManager';
+import Config               from './config/Config';
+import ArchiveHelper        from './shared/helpers/ArchiveHelper';
 
 
+(async () => {
+    const execAsync = util.promisify(exec);
+
     HttpManager.registerAxiosInterceptor();
 
+    console.log(Styles.APP_NAME(Config.appName));
+
+    /*
+     //////////////////////////////////////////////////////////////////////////////////////////////////////////// Step 1 & 2:
+     -   Read the dojo enonce file from the enonce repository
+     -   Download immutables files (maybe throw or show an error if the files have been modified ?)
+     */
+    console.log(Styles.INFO(`${ Icon.INFO }️ Checking the exercice's enonce and his immutable files`));
+    const exerciceEnonce = await DojoBackendManager.getExerciceEnonce();
+    if ( !exerciceEnonce ) {
+        console.error(Styles.ERROR(`${ Icon.ERROR } Error while getting the exercice's enonce`));
+        process.exit(ExerciceCheckerError.EXERCICE_ENONCE_GET_ERROR);
+    }
+
+    exerciceEnonce.immutable.forEach(immutableFile => {
+        const filePath = path.join(Config.folders.project, immutableFile.file_path);
+        fs.mkdirSync(path.dirname(filePath), { recursive: true });
+        fs.writeFileSync(filePath, immutableFile.content, { encoding: 'base64' });
+    });
+
+
+    /*
+     //////////////////////////////////////////////////////////////////////////////////////////////////////////// Step 3 & 4 & 5:
+     - Get override of docker-compose file (for override the volume by a bind mount to the results folder shared between dind and the host)
+     - Run docker-compose file
+     - Get logs from linked services
+     */
+    console.log(Styles.INFO(`${ Icon.INFO } Run docker compose file`));
+    const dockerComposeOverride = fs.readFileSync(path.join(__dirname, '../assets/docker-compose-override.yml'), 'utf8').replace('{{VOLUME_NAME}}', exerciceEnonce.enonceFile.result.volume).replace('{{MOUNT_PATH}}', Config.folders.resultsExercice);
+    fs.writeFileSync(`${ Config.folders.project }/docker-compose-override.yml`, dockerComposeOverride);
+
+    const changeDirectoryCommand = `cd "${ Config.folders.project }"`;
+    const dockerComposeCommand = `docker compose --project-name ${ Config.dockerCompose.projectName } --progress plain --file docker-compose.yml --file docker-compose-override.yml`;
+
+    const containerExitStatus = await new Promise<[ number, string ]>((resolve) => {
+        let logs = '####################################################### Docker Compose & Main Container Logs #######################################################\n';
+
+        const dockerCompose = spawn(`${ dockerComposeCommand } run --build ${ exerciceEnonce.enonceFile.result.container }`, {
+            cwd  : Config.folders.project,
+            shell: true,
+            env  : {
+                'DOCKER_BUILDKIT'  : '1',
+                'BUILDKIT_PROGRESS': 'plain', ...process.env
+            }
+        });
+
+        dockerCompose.stdout.on('data', (data) => {
+            logs += data.toString();
+            console.log(data.toString());
+        });
+
+        dockerCompose.stderr.on('data', (data) => {
+            logs += data.toString();
+            console.error(data.toString());
+        });
+
+        dockerCompose.on('exit', (code) => {
+            logs += '####################################################### Other Services Logs #######################################################\n';
+            resolve([ code ?? ExerciceCheckerError.DOCKER_COMPOSE_UP_ERROR, logs ]);
+        });
+    });
+    const containerExitCode = containerExitStatus[0];
+    if ( containerExitCode === ExerciceCheckerError.DOCKER_COMPOSE_UP_ERROR ) {
+        console.error(Styles.ERROR(`${ Icon.ERROR } Error while running the docker compose file`));
+        process.exit(containerExitCode);
+    }
+    fs.writeFileSync(`${ Config.folders.resultsDojo }/dockerComposeLogs.txt`, containerExitStatus[1]);
+
+    console.log(Styles.INFO(`${ Icon.INFO } Acquire logs of linked services`));
+    try {
+        await execAsync(`${ changeDirectoryCommand };${ dockerComposeCommand } logs --timestamps >> ${ Config.folders.resultsDojo }/dockerComposeLogs.txt`);
+    } catch ( error ) {
+        console.error(Styles.ERROR(`${ Icon.ERROR } Error while getting the linked services logs`));
+        process.exit(ExerciceCheckerError.DOCKER_COMPOSE_LOGS_ERROR);
+    }
+
+
+    //////////////////////////////////////////////////////////////////////////////////////////////////////////// Step 6: Check content requirements and content size
+    console.log(Styles.INFO(`${ Icon.INFO } Validating results folder size`));
+    const resultsFolderSize = await Toolbox.fs.getTotalSize(Config.folders.resultsExercice);
+    if ( resultsFolderSize > Config.resultsFolderMaxSizeInBytes ) {
+        console.error(Styles.ERROR(`${ Icon.ERROR } Results folder size is too big (bigger than ${ Config.resultsFolderMaxSizeInBytes / 1000000 })`));
+        process.exit(ExerciceCheckerError.EXERCICE_RESULTS_FOLDER_TOO_BIG);
+    }
+
+    console.log(Styles.INFO(`${ Icon.INFO } Checking results file`));
+    const resultsFileOriginPath = path.join(Config.folders.resultsExercice, Config.filenames.results);
+    const resultsFilePath = path.join(Config.folders.resultsDojo, Config.filenames.results);
+
+    if ( !fs.existsSync(resultsFileOriginPath) ) {
+        console.error(Styles.ERROR(`${ Icon.ERROR } Results file not found.`));
+        process.exit(ExerciceCheckerError.EXERCICE_RESULTS_FILE_NOT_FOUND);
+    }
+
+    fs.moveSync(resultsFileOriginPath, resultsFilePath, { overwrite: true });
+
+    const validationResults = ExerciceHelper.validateResultFile(resultsFilePath);
+
+    if ( !validationResults.isValid ) {
+        console.error(Styles.ERROR(`${ Icon.ERROR } Results file is not valid. Here are the errors :`));
+        console.error(Styles.ERROR(JSON.stringify(validationResults.errors)));
+        process.exit(ExerciceCheckerError.EXERCICE_RESULTS_FILE_SCHEMA_NOT_VALID);
+    }
+
+
+    //////////////////////////////////////////////////////////////////////////////////////////////////////////// Step 7: Upload and show the results
+    try {
+        console.log(Styles.INFO(`${ Icon.INFO } Uploading results to the dojo server`));
+        const commit: any = {};
+        Toolbox.getKeysWithPrefix(process.env, 'CI_COMMIT_').forEach(key => {
+            commit[Toolbox.snakeToCamel(key.replace('CI_COMMIT_', ''))] = process.env[key];
+        });
+
+        const files = await RecursiveFilesStats.explore(Config.folders.resultsVolume, {
+            replacePathByRelativeOne: true,
+            liteStats               : true
+        });
+
+        await DojoBackendManager.sendResults(containerExitCode, commit, validationResults.results!, files, await ArchiveHelper.getBase64(Config.folders.resultsVolume));
+    } catch ( error ) {
+        console.error(Styles.ERROR(`${ Icon.ERROR } Error while uploading the results`));
+        console.error(JSON.stringify(error));
+        process.exit(ExerciceCheckerError.UPLOAD);
+    }
+
+
+    //////////////////////////////////////////////////////////////////////////////////////////////////////////// Step 8: Exit with container exit code
+    const finalLogGlobalResult = `${ Styles.INFO('Global result') } : ${ validationResults.results!.success ? Styles.SUCCESS(`${ Icon.SUCCESS } Success`) : Styles.FAILURE(`${ Icon.FAILURE } Failure`) }`;
+
+    const finalLogExecutionExitCode = `${ Styles.INFO('Execution exit code') } : ${ (containerExitCode == 0 ? Styles.SUCCESS : Styles.ERROR)(containerExitCode) }`;
+
+    const finalLogResultNumbers = validationResults.results!.successfulTests || validationResults.results!.failedTests ? `\n\n${ Styles.SUCCESS('Tests passed') } : ${ validationResults.results!.successfulTests ?? '--' }\n${ Styles.ERROR('Tests failed') } : ${ validationResults.results!.failedTests ?? '--' }` : '';
 
-console.log(chalk.blue('Dojo Exercice Checker'));
+    const finalLogSuccessResultDetails = (validationResults.results!.successfulTestsList ?? []).map(testName => `- ${ Icon.SUCCESS } ${ testName }`).join('\n');
+    const finalLogFailedResultDetails = (validationResults.results!.failedTestsList ?? []).map(testName => `- ${ Icon.FAILURE } ${ testName }`).join('\n');
+    const finalLogResultDetails = validationResults.results!.successfulTestsList || validationResults.results!.failedTestsList ? `\n\n${ Styles.INFO('Tests') } :${ finalLogSuccessResultDetails != '' ? '\n' + finalLogSuccessResultDetails : '' }${ finalLogFailedResultDetails != '' ? '\n' + finalLogFailedResultDetails : '' }` : '';
 
+    console.log(boxen(`${ finalLogGlobalResult }\n\n${ finalLogExecutionExitCode }${ finalLogResultNumbers }${ finalLogResultDetails }`, {
+        title         : 'Results',
+        titleAlignment: 'center',
+        borderColor   : 'yellow',
+        borderStyle   : 'bold',
+        margin        : 1,
+        padding       : 1,
+        textAlignment : 'left'
+    }));
 
-// Step 1: Read the dojo enonce file from the enonce repository
-// Step 2: Download immutables files (maybe throw or show an error if the files have been modified ?) - Can be merged with step 1
-// Step 3: Run docker-compose file
-// Step 4: Wait the end of the execution of the result container
-// Step 5: Get the result from the volume
-// Step 6: Check content requirements and content size
-// Step 7: Upload and show the results
\ No newline at end of file
+    process.exit(containerExitCode);
+})();
\ No newline at end of file

ExerciceChecker/src/config/Config.ts

+import fs   from 'fs-extra';
+import path from 'path';
+
+
+class Config {
+    public readonly appName: string;
+
+    public readonly resultsFolderMaxSizeInBytes: number;
+
+    public readonly folders: {
+        project: string; resultsVolume: string; resultsDojo: string; resultsExercice: string;
+    };
+
+    public readonly filenames: {
+        results: string;
+    };
+
+    public readonly exercice: {
+        id: string; secret: string;
+    };
+
+    public readonly dockerCompose: {
+        projectName: string
+    };
+
+    constructor() {
+        this.appName = process.env.APP_NAME || '';
+
+        this.resultsFolderMaxSizeInBytes = Number(process.env.RESULTS_FOLDER_MAX_SIZE_IN_BYTES || 0);
+
+        this.folders = {
+            project        : process.env.FILES_FOLDER?.convertWithEnvVars() ?? './',
+            resultsVolume  : process.env.RESULTS_VOLUME?.convertWithEnvVars() ?? '',
+            resultsDojo    : path.join(process.env.RESULTS_VOLUME?.convertWithEnvVars() ?? '', 'Dojo/'),
+            resultsExercice: path.join(process.env.RESULTS_VOLUME?.convertWithEnvVars() ?? '', 'Exercice/')
+        };
+        this.resetResultsVolume();
+
+        this.filenames = {
+            results: process.env.RESULTS_FILENAME || ''
+        };
+
+        this.exercice = {
+            id    : process.env.DOJO_EXERCICE_ID || '',
+            secret: process.env.DOJO_SECRET || ''
+        };
+
+        this.dockerCompose = {
+            projectName: process.env.DOCKER_COMPOSE_PROJECT_NAME || ''
+        };
+    }
+
+    private resetResultsVolume(): void {
+        fs.emptyDirSync(this.folders.resultsVolume);
+        fs.emptyDirSync(this.folders.resultsDojo);
+        fs.emptyDirSync(this.folders.resultsExercice);
+    }
+}
+
+
+export default new Config();

ExerciceChecker/src/managers/DojoBackendManager.ts

 import ClientsSharedConfig from '../sharedByClients/config/ClientsSharedConfig';
 import ApiRoutes           from '../sharedByClients/types/ApiRoutes';
+import axios               from 'axios';
+import DojoResponse        from '../shared/types/Dojo/DojoResponse';
+import ExerciceEnonce      from '../sharedByClients/models/ExerciceEnonce';
+import Config              from '../config/Config';
+import ExerciceResultsFile from '../shared/types/Dojo/ExerciceResultsFile';
 
 
 class DojoBackendManager {
     public getApiUrl(route: ApiRoutes): string {
         return `${ ClientsSharedConfig.apiURL }${ route }`;
     }
+
+    public async getExerciceEnonce(): Promise<ExerciceEnonce | undefined> {
+        try {
+            return (await axios.get<DojoResponse<ExerciceEnonce>>(this.getApiUrl(ApiRoutes.EXERCICE_ENONCE).replace('{{id}}', Config.exercice.id))).data.data;
+        } catch ( error ) {
+            return undefined;
+        }
+    }
+
+    public async sendResults(exitCode: number, commit: any, results: ExerciceResultsFile, files: any, archiveBase64: string): Promise<void> {
+        try {
+            await axios.post(this.getApiUrl(ApiRoutes.EXERCICE_RESULTS).replace('{{id}}', Config.exercice.id), {
+                exitCode     : exitCode,
+                commit       : JSON.stringify(commit),
+                results      : JSON.stringify(results),
+                files        : JSON.stringify(files),
+                archiveBase64: archiveBase64
+            });
+        } catch ( error ) {
+            throw error;
+        }
+    }
 }
 
 

ExerciceChecker/src/managers/HttpManager.ts

 import axios, { AxiosRequestHeaders } from 'axios';
 import FormData                       from 'form-data';
 import ClientsSharedConfig            from '../sharedByClients/config/ClientsSharedConfig';
+import Config                         from '../config/Config';
 
 
 class HttpManager {
@@ -18,9 +19,13 @@ class HttpManager {
             }
 
             if ( config.url && (config.url.indexOf(ClientsSharedConfig.apiURL) !== -1) ) {
+                config.headers['Accept-Encoding'] = 'gzip';
+
                 if ( config.data && Object.keys(config.data).length > 0 ) {
                     config.headers['Content-Type'] = 'multipart/form-data';
                 }
+
+                config.headers.Authorization = `ExerciceSecret ${ Config.exercice.secret }`;
             }
 
             return config;

shared @ eab5c0a5

-Subproject commit c9154d42dac81311cf1957f0d75f806737849b40
+Subproject commit eab5c0a5a32079fcb439a1ad79453611c8605536

sharedByClients @ c0f10559

-Subproject commit 8fe8e9417a527cf2182a9acc440e68b99024487e
+Subproject commit c0f105590a4332ce4d6eff046324e537e769f756

ExerciceChecker/src/types/ExerciceCheckerError.ts

+enum ExerciceCheckerError {
+    EXERCICE_ENONCE_GET_ERROR              = 200,
+    DOCKER_COMPOSE_UP_ERROR                = 201,
+    DOCKER_COMPOSE_LOGS_ERROR              = 202,
+    EXERCICE_RESULTS_FOLDER_TOO_BIG        = 203,
+    EXERCICE_RESULTS_FILE_NOT_FOUND        = 204,
+    EXERCICE_RESULTS_FILE_SCHEMA_NOT_VALID = 205,
+    UPLOAD                                 = 206
+}
+
+
+export default ExerciceCheckerError;
\ No newline at end of file

ExerciceChecker/src/types/Icon.ts

+enum Icon {
+    INFO    = 'ℹ️',
+    ERROR   = '⛔️',
+    SUCCESS = '✅',
+    FAILURE = '❌'
+}
+
+
+export default Icon;
\ No newline at end of file

ExerciceChecker/src/types/Styles.ts

+import chalk from 'chalk';
+
+
+class Styles {
+    public readonly APP_NAME = chalk.bgBlue.black.bold;
+    public readonly INFO = chalk.blue;
+    public readonly ERROR = chalk.red;
+    public readonly SUCCESS = chalk.green;
+    public readonly FAILURE = chalk.red;
+}
+
+
+export default new Styles();
\ No newline at end of file