9.19.0

Updated CHANGELOG
Merge branch 'n1073645-newMagic'
2020-03-24 11:37:30 +00:00 · 2020-03-24 11:37:26 +00:00 · 2020-03-24 11:07:20 +00:00 · 2020-03-24 11:06:37 +00:00 · 2020-03-20 14:51:40 +00:00 · 2020-03-18 16:47:53 +00:00
71 changed files with 4332 additions and 5068 deletions
@@ -11,7 +11,7 @@ before_script:
  - export NODE_OPTIONS=--max_old_space_size=2048
 script:
  - grunt lint
-  - grunt test
+  - npm test
  - grunt testnodeconsumer
  - grunt prod --msg="$COMPILE_MSG"
  - xvfb-run --server-args="-screen 0 1200x800x24" grunt testui
@@ -2,6 +2,15 @@
 All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master).


+### [9.19.0] - 2020-03-24
+- Improvements to the 'Magic' operation, allowing it to recognise more data formats and provide more accurate results [@n1073645] [@n1474335] | [#966] [b765534b](https://github.com/gchq/CyberChef/commit/b765534b8b2a0454a5132a0a52d1d8844bcbdaaa)
+
+### [9.18.0] - 2020-03-13
+- 'Convert to NATO alphabet' operation added [@MarvinJWendt] | [#674]
+
+### [9.17.0] - 2020-03-13
+- 'Generate Image' operation added [@pointhi] | [#683]
+
 ### [9.16.0] - 2020-03-06
 - 'Colossus' operation added [@VirtualColossus] | [#917]

@@ -212,6 +221,9 @@ All major and minor version changes will be documented in this file. Details of



+[9.19.0]: https://github.com/gchq/CyberChef/releases/tag/v9.19.0
+[9.18.0]: https://github.com/gchq/CyberChef/releases/tag/v9.18.0
+[9.17.0]: https://github.com/gchq/CyberChef/releases/tag/v9.17.0
 [9.16.0]: https://github.com/gchq/CyberChef/releases/tag/v9.16.0
 [9.15.0]: https://github.com/gchq/CyberChef/releases/tag/v9.15.0
 [9.14.0]: https://github.com/gchq/CyberChef/releases/tag/v9.14.0
@@ -304,6 +316,8 @@ All major and minor version changes will be documented in this file. Details of
 [@cbeuw]: https://github.com/cbeuw
 [@matthieuxyz]: https://github.com/matthieuxyz
 [@Flavsditz]: https://github.com/Flavsditz
+[@pointhi]: https://github.com/pointhi
+[@MarvinJWendt]: https://github.com/MarvinJWendt

 [#95]: https://github.com/gchq/CyberChef/pull/299
 [#173]: https://github.com/gchq/CyberChef/pull/173
@@ -366,9 +380,12 @@ All major and minor version changes will be documented in this file. Details of
 [#627]: https://github.com/gchq/CyberChef/pull/627
 [#632]: https://github.com/gchq/CyberChef/pull/632
 [#653]: https://github.com/gchq/CyberChef/pull/653
+[#674]: https://github.com/gchq/CyberChef/pull/674
+[#683]: https://github.com/gchq/CyberChef/pull/683
 [#865]: https://github.com/gchq/CyberChef/pull/865
 [#912]: https://github.com/gchq/CyberChef/pull/912
 [#917]: https://github.com/gchq/CyberChef/pull/917
 [#948]: https://github.com/gchq/CyberChef/pull/948
 [#952]: https://github.com/gchq/CyberChef/pull/952
 [#965]: https://github.com/gchq/CyberChef/pull/965
+[#966]: https://github.com/gchq/CyberChef/pull/966
@@ -36,11 +36,10 @@ module.exports = function (grunt) {
            "clean:node", "clean:config", "clean:nodeConfig", "exec:generateConfig", "exec:generateNodeIndex"
        ]);

-    grunt.registerTask("test",
-        "A task which runs all the operation tests in the tests directory.",
+    grunt.registerTask("configTests",
+        "A task which configures config files in preparation for tests to be run. Use `npm tests` to run tests.",
        [
-            "clean:config", "clean:nodeConfig", "exec:generateConfig", "exec:generateNodeIndex",
-            "exec:nodeTests", "exec:opTests"
+            "clean:config", "clean:nodeConfig", "exec:generateConfig", "exec:generateNodeIndex"
        ]);

    grunt.registerTask("testui",
@@ -55,7 +54,6 @@ module.exports = function (grunt) {
        "Lints the code base",
        ["eslint", "exec:repoSize"]);

-    grunt.registerTask("tests", "test");
    grunt.registerTask("lint", "eslint");

    grunt.registerTask("findModules",
@@ -385,15 +383,9 @@ module.exports = function (grunt) {
                ]),
                sync: true
            },
-            opTests: {
-                command: "node --experimental-modules --no-warnings --no-deprecation tests/operations/index.mjs"
-            },
            browserTests: {
                command: "./node_modules/.bin/nightwatch --env prod"
            },
-            nodeTests: {
-                command: "node --experimental-modules --no-warnings --no-deprecation tests/node/index.mjs"
-            },
            setupNodeConsumers: {
                command: chainCommands([
                    "echo '\n--- Testing node consumers ---'",
@@ -1,6 +1,6 @@
 {
  "name": "cyberchef",
-  "version": "9.16.1",
+  "version": "9.19.0",
  "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
  "author": "n1474335 <n1474335@gmail.com>",
  "homepage": "https://gchq.github.io/CyberChef",
@@ -36,21 +36,22 @@
    "node >= 10"
  ],
  "devDependencies": {
-    "@babel/core": "^7.7.5",
-    "@babel/plugin-transform-runtime": "^7.7.6",
-    "@babel/preset-env": "^7.7.6",
-    "autoprefixer": "^9.7.3",
-    "babel-eslint": "^10.0.3",
+    "@babel/core": "^7.8.7",
+    "@babel/plugin-transform-runtime": "^7.8.3",
+    "@babel/preset-env": "^7.8.7",
+    "autoprefixer": "^9.7.4",
+    "babel-eslint": "^10.1.0",
    "babel-loader": "^8.0.6",
    "babel-plugin-dynamic-import-node": "^2.3.0",
    "chromedriver": "^80.0.1",
+    "cli-progress": "^3.6.0",
    "colors": "^1.4.0",
-    "copy-webpack-plugin": "^5.0.5",
-    "css-loader": "^3.2.1",
-    "eslint": "^6.7.2",
+    "copy-webpack-plugin": "^5.1.1",
+    "css-loader": "^3.4.2",
+    "eslint": "^6.8.0",
    "exports-loader": "^0.7.0",
-    "file-loader": "^5.0.2",
-    "grunt": "^1.0.4",
+    "file-loader": "^6.0.0",
+    "grunt": "^1.1.0",
    "grunt-accessibility": "~6.0.0",
    "grunt-chmod": "~1.1.1",
    "grunt-concurrent": "^3.0.0",
@@ -64,29 +65,29 @@
    "grunt-zip": "^0.18.2",
    "html-webpack-plugin": "^3.2.0",
    "imports-loader": "^0.8.0",
-    "mini-css-extract-plugin": "^0.8.0",
+    "mini-css-extract-plugin": "^0.9.0",
    "nightwatch": "^1.3.4",
-    "node-sass": "^4.13.0",
+    "node-sass": "^4.13.1",
    "postcss-css-variables": "^0.14.0",
    "postcss-import": "^12.0.1",
    "postcss-loader": "^3.0.0",
    "prompt": "^1.0.0",
-    "sass-loader": "^8.0.0",
-    "sitemap": "^5.1.0",
-    "style-loader": "^1.0.1",
-    "svg-url-loader": "^3.0.3",
-    "url-loader": "^3.0.0",
-    "webpack": "^4.41.2",
-    "webpack-bundle-analyzer": "^3.6.0",
-    "webpack-dev-server": "^3.9.0",
+    "sass-loader": "^8.0.2",
+    "sitemap": "^6.1.0",
+    "style-loader": "^1.1.3",
+    "svg-url-loader": "^5.0.0",
+    "url-loader": "^4.0.0",
+    "webpack": "^4.42.0",
+    "webpack-bundle-analyzer": "^3.6.1",
+    "webpack-dev-server": "^3.10.3",
    "webpack-node-externals": "^1.7.2",
    "worker-loader": "^2.0.0"
  },
  "dependencies": {
-    "@babel/polyfill": "^7.7.0",
-    "@babel/runtime": "^7.7.6",
+    "@babel/polyfill": "^7.8.7",
+    "@babel/runtime": "^7.8.7",
    "arrive": "^2.4.1",
-    "avsc": "^5.4.16",
+    "avsc": "^5.4.19",
    "babel-plugin-transform-builtin-extend": "1.1.2",
    "bcryptjs": "^2.4.3",
    "bignumber.js": "^9.0.0",
@@ -94,26 +95,26 @@
    "bootstrap": "4.4.1",
    "bootstrap-colorpicker": "^3.2.0",
    "bootstrap-material-design": "^4.1.2",
-    "bson": "^4.0.2",
+    "bson": "^4.0.3",
    "chi-squared": "^1.1.0",
    "codepage": "^1.14.0",
-    "core-js": "^3.4.8",
+    "core-js": "^3.6.4",
    "crypto-api": "^0.8.5",
-    "crypto-js": "^3.1.9-1",
+    "crypto-js": "^4.0.0",
    "ctph.js": "0.0.5",
-    "d3": "^5.14.2",
+    "d3": "^5.15.0",
    "d3-hexbin": "^0.2.2",
-    "diff": "^4.0.1",
-    "es6-promisify": "^6.0.2",
-    "escodegen": "^1.12.0",
+    "diff": "^4.0.2",
+    "es6-promisify": "^6.1.0",
+    "escodegen": "^1.14.1",
    "esm": "^3.2.25",
    "esmangle": "^1.0.1",
    "esprima": "^4.0.1",
    "exif-parser": "^0.1.12",
    "file-saver": "^2.0.2",
    "geodesy": "^1.1.3",
-    "highlight.js": "^9.16.2",
-    "jimp": "^0.9.3",
+    "highlight.js": "^9.18.1",
+    "jimp": "^0.9.5",
    "jquery": "3.4.1",
    "js-crc": "^0.2.0",
    "js-sha3": "^0.8.0",
@@ -122,44 +123,44 @@
    "jsonwebtoken": "^8.5.1",
    "jsqr": "^1.2.0",
    "jsrsasign": "8.0.12",
-    "kbpgp": "2.1.6",
+    "kbpgp": "2.1.13",
    "libbzip2-wasm": "0.0.4",
    "libyara-wasm": "^1.0.1",
    "lodash": "^4.17.15",
-    "loglevel": "^1.6.6",
+    "loglevel": "^1.6.7",
    "loglevel-message-prefix": "^3.0.0",
    "markdown-it": "^10.0.0",
    "moment": "^2.24.0",
-    "moment-timezone": "^0.5.27",
+    "moment-timezone": "^0.5.28",
    "ngeohash": "^0.6.3",
    "node-forge": "^0.9.1",
    "node-md6": "^0.1.0",
    "nodom": "^2.4.0",
-    "notepack.io": "^2.2.0",
+    "notepack.io": "^2.3.0",
    "nwmatcher": "^1.4.4",
    "otp": "^0.1.3",
-    "popper.js": "^1.16.0",
+    "popper.js": "^1.16.1",
    "qr-image": "^3.2.0",
    "scryptsy": "^2.1.0",
    "snackbarjs": "^1.1.0",
-    "sortablejs": "^1.10.1",
+    "sortablejs": "^1.10.2",
    "split.js": "^1.5.11",
    "ssdeep.js": "0.0.2",
-    "tesseract.js": "^2.0.0-alpha.15",
-    "ua-parser-js": "^0.7.20",
+    "tesseract.js": "^2.0.2",
+    "ua-parser-js": "^0.7.21",
    "unorm": "^1.6.0",
    "utf8": "^3.0.0",
    "vkbeautify": "^0.99.3",
-    "xmldom": "^0.1.27",
+    "xmldom": "^0.3.0",
    "xpath": "0.0.27",
-    "xregexp": "^4.2.4",
+    "xregexp": "^4.3.0",
    "zlibjs": "^0.3.1"
  },
  "scripts": {
    "start": "grunt dev",
    "build": "grunt prod",
    "repl": "node src/node/repl.js",
-    "test": "grunt test",
+    "test": "grunt configTests && node --experimental-modules --no-warnings --no-deprecation tests/node/index.mjs && node --experimental-modules --no-warnings --no-deprecation tests/operations/index.mjs",
    "test-node-consumer": "grunt testnodeconsumer",
    "testui": "grunt testui",
    "testuidev": "npx nightwatch --env=dev",
@@ -39,7 +39,7 @@ class Chef {
    */
    async bake(input, recipeConfig, options) {
        log.debug("Chef baking");
-        const startTime = new Date().getTime(),
+        const startTime = Date.now(),
            recipe      = new Recipe(recipeConfig),
            containsFc  = recipe.containsFlowControl(),
            notUTF8     = options && "treatAsUtf8" in options && !options.treatAsUtf8;
@@ -84,7 +84,7 @@ class Chef {
            result: await this.dish.get(returnType, notUTF8),
            type: Dish.enumLookup(this.dish.type),
            progress: progress,
-            duration: new Date().getTime() - startTime,
+            duration: Date.now() - startTime,
            error: error
        };
    }
@@ -110,7 +110,7 @@ class Chef {
    silentBake(recipeConfig) {
        log.debug("Running silent bake");

-        const startTime = new Date().getTime(),
+        const startTime = Date.now(),
            recipe = new Recipe(recipeConfig),
            dish = new Dish();

@@ -119,7 +119,7 @@ class Chef {
        } catch (err) {
            // Suppress all errors
        }
-        return new Date().getTime() - startTime;
+        return Date.now() - startTime;
    }


@@ -146,7 +146,12 @@ class Chef {
            const func = direction === "forward" ? highlights[i].f : highlights[i].b;

            if (typeof func == "function") {
-                pos = func(pos, highlights[i].args);
+                try {
+                    pos = func(pos, highlights[i].args);
+                } catch (err) {
+                    // Throw away highlighting errors
+                    pos = [];
+                }
            }
        }

@@ -1182,6 +1182,7 @@ class Utils {
            "CRLF":          /\r\n/g,
            "Forward slash": /\//g,
            "Backslash":     /\\/g,
+            "0x with comma": /,?0x/g,
            "0x":            /0x/g,
            "\\x":           /\\x/g,
            "None":          /\s+/g // Included here to remove whitespace when there shouldn't be any
@@ -1335,7 +1336,7 @@ export function sendStatusMessage(msg) {
        self.sendStatusMessage(msg);
    else if (isWebEnvironment())
        app.alert(msg, 10000);
-    else if (isNodeEnvironment())
+    else if (isNodeEnvironment() && !global.TESTING)
        // eslint-disable-next-line no-console
        console.debug(msg);
 }
@@ -201,7 +201,8 @@
            "Encode text",
            "Decode text",
            "Remove Diacritics",
-            "Unescape Unicode Characters"
+            "Unescape Unicode Characters",
+            "Convert to NATO alphabet"
        ]
    },
    {
@@ -395,6 +396,7 @@
        "ops": [
            "Render Image",
            "Play Media",
+            "Generate Image",
            "Optical Character Recognition",
            "Remove EXIF",
            "Extract EXIF",
@@ -42,13 +42,10 @@ for (const opObj in Ops) {
        outputType:  op.presentType,
        flowControl: op.flowControl,
        manualBake:  op.manualBake,
-        args:        op.args
+        args:        op.args,
+        checks:      op.checks
    };

-    if ("patterns" in op) {
-        operationConfig[op.name].patterns = op.patterns;
-    }
-
    if (!(op.module in modules))
        modules[op.module] = {};
    modules[op.module][op.name] = opObj;
@@ -1724,6 +1724,25 @@ export const FILE_SIGNATURES = {
            },
            extractor: null
        },
+        {
+            name: "Jar Archive",
+            extension: "jar",
+            mime: "application/java-archive",
+            description: "",
+            signature: {
+                0: 0x50,
+                1: 0x4B,
+                2: 0x03,
+                3: 0x04,
+                4: 0x14,
+                5: 0x00,
+                6: 0x08,
+                7: 0x00,
+                8: 0x08,
+                9: 0x00
+            },
+            extractor: extractZIP
+        },
        {
            name: "lzop compressed",
            extension: "lzop,lzo",
@@ -1742,7 +1761,7 @@ export const FILE_SIGNATURES = {
            extractor: extractLZOP
        },
        {
-            name: "Linux deb",
+            name: "Linux deb package",
            extension: "deb",
            mime: "application/vnd.debian.binary-package",
            description: "",
@@ -1755,7 +1774,7 @@ export const FILE_SIGNATURES = {
                5: 0x68,
                6: 0x3e
            },
-            extractor: null
+            extractor: extractDEB
        },
        {
            name: "Apple Disk Image",
@@ -3448,6 +3467,37 @@ export function extractXZ(bytes, offset) {
 }


+/**
+ * DEB extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {Number} offset
+ */
+export function extractDEB(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Move past !<arch>
+    stream.moveForwardsBy(8);
+    while (stream.hasMore()) {
+
+        // Move to size field.
+        stream.moveForwardsBy(48);
+        let fsize= "";
+
+        // Convert size to a usable number.
+        for (const elem of stream.getBytes(10)) {
+            fsize += String.fromCharCode(elem);
+        }
+        fsize = parseInt(fsize.trim(), 10);
+
+        // Move past `\n
+        stream.moveForwardsBy(2);
+        stream.moveForwardsBy(fsize);
+    }
+    return stream.carve();
+}
+
+
 /**
 * ELF extractor.
 *
@@ -26,7 +26,7 @@ export function ipv4CidrRange(cidr, includeNetworkInfo, enumerateAddresses, allo
    let output = "";

    if (cidrRange < 0 || cidrRange > 31) {
-        return "IPv4 CIDR must be less than 32";
+        throw new OperationError("IPv4 CIDR must be less than 32");
    }

    const mask = ~(0xFFFFFFFF >>> cidrRange),
@@ -64,7 +64,7 @@ export function ipv6CidrRange(cidr, includeNetworkInfo) {
        cidrRange = parseInt(cidr[cidr.length-1], 10);

    if (cidrRange < 0 || cidrRange > 127) {
-        return "IPv6 CIDR must be less than 128";
+        throw new OperationError("IPv6 CIDR must be less than 128");
    }

    const ip1 = new Array(8),
@@ -211,7 +211,7 @@ export function ipv4ListedRange(match, includeNetworkInfo, enumerateAddresses, a
        const network = strToIpv4(ipv4CidrList[i].split("/")[0]);
        const cidrRange = parseInt(ipv4CidrList[i].split("/")[1], 10);
        if (cidrRange < 0 || cidrRange > 31) {
-            return "IPv4 CIDR must be less than 32";
+            throw new OperationError("IPv4 CIDR must be less than 32");
        }
        const mask = ~(0xFFFFFFFF >>> cidrRange),
            cidrIp1 = network & mask,
@@ -254,7 +254,7 @@ export function ipv6ListedRange(match, includeNetworkInfo) {
        const cidrRange = parseInt(ipv6CidrList[i].split("/")[1], 10);

        if (cidrRange < 0 || cidrRange > 127) {
-            return "IPv6 CIDR must be less than 128";
+            throw new OperationError("IPv6 CIDR must be less than 128");
        }

        const cidrIp1 = new Array(8),
@@ -2,7 +2,7 @@ import OperationConfig from "../config/OperationConfig.json";
 import Utils, { isWorkerEnvironment } from "../Utils.mjs";
 import Recipe from "../Recipe.mjs";
 import Dish from "../Dish.mjs";
-import {detectFileType} from "./FileType.mjs";
+import {detectFileType, isType} from "./FileType.mjs";
 import chiSquared from "chi-squared";

 /**
@@ -19,31 +19,38 @@ class Magic {
     * Magic constructor.
     *
     * @param {ArrayBuffer} buf
-     * @param {Object[]} [opPatterns]
+     * @param {Object[]} [opCriteria]
+     * @param {Object} [prevOp]
     */
-    constructor(buf, opPatterns) {
+    constructor(buf, opCriteria=Magic._generateOpCriteria(), prevOp=null) {
        this.inputBuffer = new Uint8Array(buf);
        this.inputStr = Utils.arrayBufferToStr(buf);
-        this.opPatterns = opPatterns || Magic._generateOpPatterns();
+        this.opCriteria = opCriteria;
+        this.prevOp = prevOp;
    }

    /**
-     * Finds operations that claim to be able to decode the input based on regular
-     * expression matches.
+     * Finds operations that claim to be able to decode the input based on various criteria.
     *
     * @returns {Object[]}
     */
-    findMatchingOps() {
-        const matches = [];
+    findMatchingInputOps() {
+        const matches = [],
+            inputEntropy = this.calcEntropy();

-        for (let i = 0; i < this.opPatterns.length; i++) {
-            const pattern = this.opPatterns[i],
-                regex = new RegExp(pattern.match, pattern.flags);
+        this.opCriteria.forEach(check => {
+            // If the input doesn't lie in the required entropy range, move on
+            if (check.entropyRange &&
+                (inputEntropy < check.entropyRange[0] ||
+                inputEntropy > check.entropyRange[1]))
+                return;
+            // If the input doesn't match the pattern, move on
+            if (check.pattern &&
+                !check.pattern.test(this.inputStr))
+                return;

-            if (regex.test(this.inputStr)) {
-                matches.push(pattern);
-            }
-        }
+            matches.push(check);
+        });

        return matches;
    }
@@ -185,8 +192,10 @@ class Magic {
     *
     * @returns {number}
     */
-    calcEntropy() {
-        const prob = this._freqDist();
+    calcEntropy(data=this.inputBuffer, standalone=false) {
+        if (!standalone && this.inputEntropy) return this.inputEntropy;
+
+        const prob = this._freqDist(data, standalone);
        let entropy = 0,
            p;

@@ -195,6 +204,8 @@ class Magic {
            if (p === 0) continue;
            entropy += p * Math.log(p) / Math.log(2);
        }
+
+        if (!standalone) this.inputEntropy = -entropy;
        return -entropy;
    }

@@ -264,25 +275,59 @@ class Magic {
        return results;
    }

+    /**
+     * Checks whether the data passes output criteria for an operation check
+     *
+     * @param {ArrayBuffer} data
+     * @param {Object} criteria
+     * @returns {boolean}
+     */
+    outputCheckPasses(data, criteria) {
+        if (criteria.pattern) {
+            const dataStr = Utils.arrayBufferToStr(data),
+                regex = new RegExp(criteria.pattern, criteria.flags);
+            if (!regex.test(dataStr))
+                return false;
+        }
+        if (criteria.entropyRange) {
+            const dataEntropy = this.calcEntropy(data, true);
+            if (dataEntropy < criteria.entropyRange[0] || dataEntropy > criteria.entropyRange[1])
+                return false;
+        }
+        if (criteria.mime &&
+            !isType(criteria.mime, data))
+            return false;
+
+        return true;
+    }
+
    /**
     * Speculatively executes matching operations, recording metadata of each result.
     *
     * @param {number} [depth=0] - How many levels to try to execute
     * @param {boolean} [extLang=false] - Extensive language support (false = only check the most
-     *                                    common Internet languages)
+     *     common Internet languages)
     * @param {boolean} [intensive=false] - Run brute-forcing on each branch (significantly affects
-     *                                      performance)
+     *     performance)
     * @param {Object[]} [recipeConfig=[]] - The recipe configuration up to this point
     * @param {boolean} [useful=false] - Whether the current recipe should be scored highly
-     * @param {string} [crib=null] - The regex crib provided by the user, for filtering the operation output
+     * @param {string} [crib=null] - The regex crib provided by the user, for filtering the operation
+     *     output
     * @returns {Object[]} - A sorted list of the recipes most likely to result in correct decoding
     */
-    async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false, crib=null) {
+    async speculativeExecution(
+        depth=0,
+        extLang=false,
+        intensive=false,
+        recipeConfig=[],
+        useful=false,
+        crib=null) {
+
+        // If we have reached the recursion depth, return
        if (depth < 0) return [];

        // Find any operations that can be run on this data
-        const matchingOps = this.findMatchingOps();
-
+        const matchingOps = this.findMatchingInputOps();
        let results = [];

        // Record the properties of the current data
@@ -308,17 +353,21 @@ class Magic {
                },
                output = await this._runRecipe([opConfig]);

-            // If the recipe is repeating and returning the same data, do not continue
-            if (prevOp && op.op === prevOp.op && _buffersEqual(output, this.inputBuffer)) {
-                return;
-            }
-
            // If the recipe returned an empty buffer, do not continue
            if (_buffersEqual(output, new ArrayBuffer())) {
                return;
            }

-            const magic = new Magic(output, this.opPatterns),
+            // If the recipe is repeating and returning the same data, do not continue
+            if (prevOp && op.op === prevOp.op && _buffersEqual(output, this.inputBuffer)) {
+                return;
+            }
+
+            // If the output criteria for this op doesn't match the output, do not continue
+            if (op.output && !this.outputCheckPasses(output, op.output))
+                return;
+
+            const magic = new Magic(output, this.opCriteria, OperationConfig[op.op]),
                speculativeResults = await magic.speculativeExecution(
                    depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, crib);

@@ -330,7 +379,7 @@ class Magic {
            const bfEncodings = await this.bruteForce();

            await Promise.all(bfEncodings.map(async enc => {
-                const magic = new Magic(enc.data, this.opPatterns),
+                const magic = new Magic(enc.data, this.opCriteria, undefined),
                    bfResults = await magic.speculativeExecution(
                        depth-1, extLang, false, [...recipeConfig, enc.conf], false, crib);

@@ -345,7 +394,8 @@ class Magic {
                r.languageScores[0].probability > 0 ||    // Some kind of language was found
                r.fileType ||                             // A file was found
                r.isUTF8 ||                               // UTF-8 was found
-                r.matchingOps.length                      // A matching op was found
+                r.matchingOps.length ||                   // A matching op was found
+                r.matchesCrib                             // The crib matches
            )
        );

@@ -376,9 +426,10 @@ class Magic {
            bScore += b.entropy;

            // A result with no recipe but matching ops suggests there are better options
-            if ((!a.recipe.length && a.matchingOps.length) &&
-                b.recipe.length)
+            if ((!a.recipe.length && a.matchingOps.length) && b.recipe.length)
                return 1;
+            if ((!b.recipe.length && b.matchingOps.length) && a.recipe.length)
+                return -1;

            return aScore - bScore;
        });
@@ -417,14 +468,16 @@ class Magic {
     * Calculates the number of times each byte appears in the input as a percentage
     *
     * @private
+     * @param {ArrayBuffer} [data]
+     * @param {boolean} [standalone]
     * @returns {number[]}
     */
-    _freqDist() {
-        if (this.freqDist) return this.freqDist;
+    _freqDist(data=this.inputBuffer, standalone=false) {
+        if (!standalone && this.freqDist) return this.freqDist;

-        const len = this.inputBuffer.length;
+        const len = data.length,
+            counts = new Array(256).fill(0);
        let i = len;
-        const counts = new Array(256).fill(0);

        if (!len) {
            this.freqDist = counts;
@@ -432,13 +485,15 @@ class Magic {
        }

        while (i--) {
-            counts[this.inputBuffer[i]]++;
+            counts[data[i]]++;
        }

-        this.freqDist = counts.map(c => {
+        const result = counts.map(c => {
            return c / len * 100;
        });
-        return this.freqDist;
+
+        if (!standalone) this.freqDist = result;
+        return result;
    }

    /**
@@ -447,24 +502,29 @@ class Magic {
     * @private
     * @returns {Object[]}
     */
-    static _generateOpPatterns() {
-        const opPatterns = [];
+    static _generateOpCriteria() {
+        const opCriteria = [];

        for (const op in OperationConfig) {
-            if (!("patterns" in OperationConfig[op])) continue;
+            if (!("checks" in OperationConfig[op]))
+                continue;

-            OperationConfig[op].patterns.forEach(pattern => {
-                opPatterns.push({
+            OperationConfig[op].checks.forEach(check => {
+                // Add to the opCriteria list.
+                // Compile the regex here and cache the compiled version so we
+                // don't have to keep calculating it.
+                opCriteria.push({
                    op: op,
-                    match: pattern.match,
-                    flags: pattern.flags,
-                    args: pattern.args,
-                    useful: pattern.useful || false
+                    pattern: check.pattern ? new RegExp(check.pattern, check.flags) : null,
+                    args: check.args,
+                    useful: check.useful,
+                    entropyRange: check.entropyRange,
+                    output: check.output
                });
            });
        }

-        return opPatterns;
+        return opCriteria;
    }

    /**
@@ -33,6 +33,38 @@ class A1Z26CipherDecode extends Operation {
                value: DELIM_OPTIONS
            }
        ];
+        this.checks = [
+            {
+                pattern:  "^\\s*([12]?[0-9] )+[12]?[0-9]\\s*$",
+                flags:  "",
+                args:   ["Space"]
+            },
+            {
+                pattern:  "^\\s*([12]?[0-9],)+[12]?[0-9]\\s*$",
+                flags:  "",
+                args:   ["Comma"]
+            },
+            {
+                pattern:  "^\\s*([12]?[0-9];)+[12]?[0-9]\\s*$",
+                flags:  "",
+                args:   ["Semi-colon"]
+            },
+            {
+                pattern:  "^\\s*([12]?[0-9]:)+[12]?[0-9]\\s*$",
+                flags:  "",
+                args:   ["Colon"]
+            },
+            {
+                pattern:  "^\\s*([12]?[0-9]\\n)+[12]?[0-9]\\s*$",
+                flags:  "",
+                args:   ["Line feed"]
+            },
+            {
+                pattern:  "^\\s*([12]?[0-9]\\r\\n)+[12]?[0-9]\\s*$",
+                flags:  "",
+                args:   ["CRLF"]
+            }
+        ];
    }

    /**
@@ -44,6 +44,48 @@ class BaconCipherDecode extends Operation {
                "value": false
            }
        ];
+        this.checks = [
+            {
+                pattern:  "^\\s*([01]{5}\\s?)+$",
+                flags:  "",
+                args:   ["Standard (I=J and U=V)", "0/1", false]
+            },
+            {
+                pattern:  "^\\s*([01]{5}\\s?)+$",
+                flags:  "",
+                args:   ["Standard (I=J and U=V)", "0/1", true]
+            },
+            {
+                pattern:  "^\\s*([AB]{5}\\s?)+$",
+                flags:  "",
+                args:   ["Standard (I=J and U=V)", "A/B", false]
+            },
+            {
+                pattern:  "^\\s*([AB]{5}\\s?)+$",
+                flags:  "",
+                args:   ["Standard (I=J and U=V)", "A/B", true]
+            },
+            {
+                pattern:  "^\\s*([01]{5}\\s?)+$",
+                flags:  "",
+                args:   ["Complete", "0/1", false]
+            },
+            {
+                pattern:  "^\\s*([01]{5}\\s?)+$",
+                flags:  "",
+                args:   ["Complete", "0/1", true]
+            },
+            {
+                pattern:  "^\\s*([AB]{5}\\s?)+$",
+                flags:  "",
+                args:   ["Complete", "A/B", false]
+            },
+            {
+                pattern:  "^\\s*([AB]{5}\\s?)+$",
+                flags:  "",
+                args:   ["Complete", "A/B", true]
+            }
+        ];
    }

    /**
@@ -33,9 +33,9 @@ class Bzip2Decompress extends Operation {
                value: false
            }
        ];
-        this.patterns = [
+        this.checks = [
            {
-                "match": "^\\x42\\x5a\\x68",
+                "pattern": "^\\x42\\x5a\\x68",
                "flags": "",
                "args": []
            }
@@ -6,7 +6,7 @@
 * @license Apache-2.0
 */

-import Operation from "../Operation";
+import Operation from "../Operation.mjs";
 import OperationError from "../errors/OperationError.mjs";
 import { ColossusComputer } from "../lib/Colossus.mjs";
 import { SWITCHES, VALID_ITA2 } from "../lib/Lorenz.mjs";
@@ -0,0 +1,82 @@
+/**
+ * @author MarvinJWendt [git@marvinjwendt.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation.mjs";
+
+/**
+ * Convert to NATO alphabet operation
+ */
+class ConvertToNATOAlphabet extends Operation {
+    /**
+     * ConvertToNATOAlphabet constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Convert to NATO alphabet";
+        this.module = "Default";
+        this.description = "Converts characters to their representation in the NATO phonetic alphabet.";
+        this.infoURL = "https://wikipedia.org/wiki/NATO_phonetic_alphabet";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        return input.replace(/[a-z0-9,/.]/ig, letter => {
+            return lookup[letter.toUpperCase()];
+        });
+    }
+}
+
+const lookup = {
+    "A": "Alfa ",
+    "B": "Bravo ",
+    "C": "Charlie ",
+    "D": "Delta ",
+    "E": "Echo ",
+    "F": "Foxtrot ",
+    "G": "Golf ",
+    "H": "Hotel ",
+    "I": "India ",
+    "J": "Juliett ",
+    "K": "Kilo ",
+    "L": "Lima ",
+    "M": "Mike ",
+    "N": "November ",
+    "O": "Oscar ",
+    "P": "Papa ",
+    "Q": "Quebec ",
+    "R": "Romeo ",
+    "S": "Sierra ",
+    "T": "Tango ",
+    "U": "Uniform ",
+    "V": "Victor ",
+    "W": "Whiskey ",
+    "X": "X-ray ",
+    "Y": "Yankee ",
+    "Z": "Zulu ",
+    "0": "Zero ",
+    "1": "One ",
+    "2": "Two ",
+    "3": "Three ",
+    "4": "Four ",
+    "5": "Five ",
+    "6": "Six ",
+    "7": "Seven ",
+    "8": "Eight ",
+    "9": "Nine ",
+    ",": "Comma ",
+    "/": "Fraction bar ",
+    ".": "Full stop ",
+};
+
+export default ConvertToNATOAlphabet;
@@ -24,6 +24,13 @@ class DechunkHTTPResponse extends Operation {
        this.inputType = "string";
        this.outputType = "string";
        this.args = [];
+        this.checks = [
+            {
+                pattern:  "^[0-9A-F]+\r\n",
+                flags:  "i",
+                args:   []
+            }
+        ];
    }

    /**
@@ -30,6 +30,13 @@ class DecodeNetBIOSName extends Operation {
                "value": 65
            }
        ];
+        this.checks = [
+            {
+                pattern:  "^\\s*\\S{32}$",
+                flags:  "",
+                args:   [65]
+            }
+        ];
    }

    /**
@@ -25,7 +25,17 @@ class DefangIPAddresses extends Operation {
        this.inputType = "string";
        this.outputType = "string";
        this.args = [];
-
+        this.checks = [
+            {
+                pattern: "^\\s*(([0-9]{1,3}\\.){3}[0-9]{1,3}|([0-9a-f]{4}:){7}[0-9a-f]{4})\\s*$",
+                flags: "i",
+                args: [],
+                output: {
+                    pattern: "^\\s*(([0-9]{1,3}\\[\\.\\]){3}[0-9]{1,3}|([0-9a-f]{4}\\[\\:\\]){7}[0-9a-f]{4})\\s*$",
+                    flags: "i"
+                }
+            }
+        ];
    }

    /**
@@ -44,22 +44,22 @@ class EscapeUnicodeCharacters extends Operation {
                "value": true
            }
        ];
-        this.patterns = [
+        this.checks = [
            {
-                match: "\\\\u(?:[\\da-f]{4,6})",
+                pattern: "\\\\u(?:[\\da-f]{4,6})",
                flags: "i",
                args: ["\\u"]
            },
            {
-                match: "%u(?:[\\da-f]{4,6})",
+                pattern: "%u(?:[\\da-f]{4,6})",
                flags: "i",
                args: ["%u"]
            },
            {
-                match: "U\\+(?:[\\da-f]{4,6})",
+                pattern: "U\\+(?:[\\da-f]{4,6})",
                flags: "i",
                args: ["U+"]
-            },
+            }
        ];
    }

@@ -49,9 +49,9 @@ class FromBCD extends Operation {
                "value": FORMAT
            }
        ];
-        this.patterns = [
+        this.checks = [
            {
-                match: "^(?:\\d{4} ){3,}\\d{4}$",
+                pattern: "^(?:\\d{4} ){3,}\\d{4}$",
                flags: "",
                args: ["8 4 2 1", true, false, "Nibbles"]
            },
@@ -36,12 +36,12 @@ class FromBase32 extends Operation {
                value: true
            }
        ];
-        this.patterns = [
+        this.checks = [
            {
-                match: "^(?:[A-Z2-7]{8})+(?:[A-Z2-7]{2}={6}|[A-Z2-7]{4}={4}|[A-Z2-7]{5}={3}|[A-Z2-7]{7}={1})?$",
+                pattern: "^(?:[A-Z2-7]{8})+(?:[A-Z2-7]{2}={6}|[A-Z2-7]{4}={4}|[A-Z2-7]{5}={3}|[A-Z2-7]{7}={1})?$",
                flags: "",
                args: ["A-Z2-7=", false]
-            },
+            }
        ];
    }

@@ -38,14 +38,14 @@ class FromBase58 extends Operation {
                "value": true
            }
        ];
-        this.patterns = [
+        this.checks = [
            {
-                match: "^[1-9A-HJ-NP-Za-km-z]{20,}$",
+                pattern: "^[1-9A-HJ-NP-Za-km-z]{20,}$",
                flags: "",
                args: ["123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz", false]
            },
            {
-                match: "^[1-9A-HJ-NP-Za-km-z]{20,}$",
+                pattern: "^[1-9A-HJ-NP-Za-km-z]{20,}$",
                flags: "",
                args: ["rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz", false]
            },
@@ -36,69 +36,69 @@ class FromBase64 extends Operation {
                value: true
            }
        ];
-        this.patterns = [
+        this.checks = [
            {
-                match: "^\\s*(?:[A-Z\\d+/]{4})+(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$",
+                pattern: "^\\s*(?:[A-Z\\d+/]{4})+(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$",
                flags: "i",
                args: ["A-Za-z0-9+/=", true]
            },
            {
-                match: "^\\s*[A-Z\\d\\-_]{20,}\\s*$",
+                pattern: "^\\s*[A-Z\\d\\-_]{20,}\\s*$",
                flags: "i",
                args: ["A-Za-z0-9-_", true]
            },
            {
-                match: "^\\s*(?:[A-Z\\d+\\-]{4}){5,}(?:[A-Z\\d+\\-]{2}==|[A-Z\\d+\\-]{3}=)?\\s*$",
+                pattern: "^\\s*(?:[A-Z\\d+\\-]{4}){5,}(?:[A-Z\\d+\\-]{2}==|[A-Z\\d+\\-]{3}=)?\\s*$",
                flags: "i",
                args: ["A-Za-z0-9+\\-=", true]
            },
            {
-                match: "^\\s*(?:[A-Z\\d./]{4}){5,}(?:[A-Z\\d./]{2}==|[A-Z\\d./]{3}=)?\\s*$",
+                pattern: "^\\s*(?:[A-Z\\d./]{4}){5,}(?:[A-Z\\d./]{2}==|[A-Z\\d./]{3}=)?\\s*$",
                flags: "i",
                args: ["./0-9A-Za-z=", true]
            },
            {
-                match: "^\\s*[A-Z\\d_.]{20,}\\s*$",
+                pattern: "^\\s*[A-Z\\d_.]{20,}\\s*$",
                flags: "i",
                args: ["A-Za-z0-9_.", true]
            },
            {
-                match: "^\\s*(?:[A-Z\\d._]{4}){5,}(?:[A-Z\\d._]{2}--|[A-Z\\d._]{3}-)?\\s*$",
+                pattern: "^\\s*(?:[A-Z\\d._]{4}){5,}(?:[A-Z\\d._]{2}--|[A-Z\\d._]{3}-)?\\s*$",
                flags: "i",
                args: ["A-Za-z0-9._-", true]
            },
            {
-                match: "^\\s*(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$",
+                pattern: "^\\s*(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$",
                flags: "i",
                args: ["0-9a-zA-Z+/=", true]
            },
            {
-                match: "^\\s*(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$",
+                pattern: "^\\s*(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$",
                flags: "i",
                args: ["0-9A-Za-z+/=", true]
            },
            {
-                match: "^[ !\"#$%&'()*+,\\-./\\d:;<=>?@A-Z[\\\\\\]^_]{20,}$",
+                pattern: "^[ !\"#$%&'()*+,\\-./\\d:;<=>?@A-Z[\\\\\\]^_]{20,}$",
                flags: "",
                args: [" -_", false]
            },
            {
-                match: "^\\s*[A-Z\\d+\\-]{20,}\\s*$",
+                pattern: "^\\s*[A-Z\\d+\\-]{20,}\\s*$",
                flags: "i",
                args: ["+\\-0-9A-Za-z", true]
            },
            {
-                match: "^\\s*[!\"#$%&'()*+,\\-0-689@A-NP-VX-Z[`a-fh-mp-r]{20,}\\s*$",
+                pattern: "^\\s*[!\"#$%&'()*+,\\-0-689@A-NP-VX-Z[`a-fh-mp-r]{20,}\\s*$",
                flags: "",
                args: ["!-,-0-689@A-NP-VX-Z[`a-fh-mp-r", true]
            },
            {
-                match: "^\\s*(?:[N-ZA-M\\d+/]{4}){5,}(?:[N-ZA-M\\d+/]{2}==|[N-ZA-M\\d+/]{3}=)?\\s*$",
+                pattern: "^\\s*(?:[N-ZA-M\\d+/]{4}){5,}(?:[N-ZA-M\\d+/]{2}==|[N-ZA-M\\d+/]{3}=)?\\s*$",
                flags: "i",
                args: ["N-ZA-Mn-za-m0-9+/=", true]
            },
            {
-                match: "^\\s*[A-Z\\d./]{20,}\\s*$",
+                pattern: "^\\s*[A-Z\\d./]{20,}\\s*$",
                flags: "i",
                args: ["./0-9A-Za-z", true]
            },
@@ -33,39 +33,39 @@ class FromBinary extends Operation {
                "value": BIN_DELIM_OPTIONS
            }
        ];
-        this.patterns = [
+        this.checks = [
            {
-                match: "^(?:[01]{8})+$",
+                pattern: "^(?:[01]{8})+$",
                flags: "",
                args: ["None"]
            },
            {
-                match: "^(?:[01]{8})(?: [01]{8})*$",
+                pattern: "^(?:[01]{8})(?: [01]{8})*$",
                flags: "",
                args: ["Space"]
            },
            {
-                match: "^(?:[01]{8})(?:,[01]{8})*$",
+                pattern: "^(?:[01]{8})(?:,[01]{8})*$",
                flags: "",
                args: ["Comma"]
            },
            {
-                match: "^(?:[01]{8})(?:;[01]{8})*$",
+                pattern: "^(?:[01]{8})(?:;[01]{8})*$",
                flags: "",
                args: ["Semi-colon"]
            },
            {
-                match: "^(?:[01]{8})(?::[01]{8})*$",
+                pattern: "^(?:[01]{8})(?::[01]{8})*$",
                flags: "",
                args: ["Colon"]
            },
            {
-                match: "^(?:[01]{8})(?:\\n[01]{8})*$",
+                pattern: "^(?:[01]{8})(?:\\n[01]{8})*$",
                flags: "",
                args: ["Line feed"]
            },
            {
-                match: "^(?:[01]{8})(?:\\r\\n[01]{8})*$",
+                pattern: "^(?:[01]{8})(?:\\r\\n[01]{8})*$",
                flags: "",
                args: ["CRLF"]
            },
@@ -36,37 +36,37 @@ class FromDecimal extends Operation {
                "value": false
            }
        ];
-        this.patterns = [
+        this.checks = [
            {
-                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?: (?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                pattern: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?: (?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                flags: "",
                args: ["Space", false]
            },
            {
-                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:,(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                pattern: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:,(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                flags: "",
                args: ["Comma", false]
            },
            {
-                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:;(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                pattern: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:;(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                flags: "",
                args: ["Semi-colon", false]
            },
            {
-                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?::(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                pattern: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?::(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                flags: "",
                args: ["Colon", false]
            },
            {
-                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                pattern: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                flags: "",
                args: ["Line feed", false]
            },
            {
-                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\r\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                pattern: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\r\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
                flags: "",
                args: ["CRLF", false]
-            },
+            }
        ];
    }

@@ -25,12 +25,12 @@ class FromHTMLEntity extends Operation {
        this.inputType = "string";
        this.outputType = "string";
        this.args = [];
-        this.patterns = [
+        this.checks = [
            {
-                match: "&(?:#\\d{2,3}|#x[\\da-f]{2}|[a-z]{2,6});",
+                pattern: "&(?:#\\d{2,3}|#x[\\da-f]{2}|[a-z]{2,6});",
                flags: "i",
                args: []
-            },
+            }
        ];
    }

@@ -32,49 +32,54 @@ class FromHex extends Operation {
                value: FROM_HEX_DELIM_OPTIONS
            }
        ];
-        this.patterns = [
+        this.checks = [
            {
-                match: "^(?:[\\dA-F]{2})+$",
+                pattern: "^(?:[\\dA-F]{2})+$",
                flags: "i",
                args: ["None"]
            },
            {
-                match: "^[\\dA-F]{2}(?: [\\dA-F]{2})*$",
+                pattern: "^[\\dA-F]{2}(?: [\\dA-F]{2})*$",
                flags: "i",
                args: ["Space"]
            },
            {
-                match: "^[\\dA-F]{2}(?:,[\\dA-F]{2})*$",
+                pattern: "^[\\dA-F]{2}(?:,[\\dA-F]{2})*$",
                flags: "i",
                args: ["Comma"]
            },
            {
-                match: "^[\\dA-F]{2}(?:;[\\dA-F]{2})*$",
+                pattern: "^[\\dA-F]{2}(?:;[\\dA-F]{2})*$",
                flags: "i",
                args: ["Semi-colon"]
            },
            {
-                match: "^[\\dA-F]{2}(?::[\\dA-F]{2})*$",
+                pattern: "^[\\dA-F]{2}(?::[\\dA-F]{2})*$",
                flags: "i",
                args: ["Colon"]
            },
            {
-                match: "^[\\dA-F]{2}(?:\\n[\\dA-F]{2})*$",
+                pattern: "^[\\dA-F]{2}(?:\\n[\\dA-F]{2})*$",
                flags: "i",
                args: ["Line feed"]
            },
            {
-                match: "^[\\dA-F]{2}(?:\\r\\n[\\dA-F]{2})*$",
+                pattern: "^[\\dA-F]{2}(?:\\r\\n[\\dA-F]{2})*$",
                flags: "i",
                args: ["CRLF"]
            },
            {
-                match: "^[\\dA-F]{2}(?:0x[\\dA-F]{2})*$",
+                pattern: "^(?:0x[\\dA-F]{2})+$",
                flags: "i",
                args: ["0x"]
            },
            {
-                match: "^[\\dA-F]{2}(?:\\\\x[\\dA-F]{2})*$",
+                pattern: "^0x[\\dA-F]{2}(?:,0x[\\dA-F]{2})*$",
+                flags: "i",
+                args: ["0x with comma"]
+            },
+            {
+                pattern: "^(?:\\\\x[\\dA-F]{2})+$",
                flags: "i",
                args: ["\\x"]
            }
@@ -26,6 +26,13 @@ class FromHexContent extends Operation {
        this.inputType = "string";
        this.outputType = "byteArray";
        this.args = [];
+        this.checks = [
+            {
+                pattern:  "\\|([\\da-f]{2} ?)+\\|",
+                flags:  "i",
+                args:   []
+            }
+        ];
    }

    /**
@@ -27,9 +27,9 @@ class FromHexdump extends Operation {
        this.inputType = "string";
        this.outputType = "byteArray";
        this.args = [];
-        this.patterns = [
+        this.checks = [
            {
-                match: "^(?:(?:[\\dA-F]{4,16}h?:?)?[ \\t]*((?:[\\dA-F]{2} ){1,8}(?:[ \\t]|[\\dA-F]{2}-)(?:[\\dA-F]{2} ){1,8}|(?:[\\dA-F]{4} )*[\\dA-F]{4}|(?:[\\dA-F]{2} )*[\\dA-F]{2})[^\\n]*\\n?){2,}$",
+                pattern: "^(?:(?:[\\dA-F]{4,16}h?:?)?[ \\t]*((?:[\\dA-F]{2} ){1,8}(?:[ \\t]|[\\dA-F]{2}-)(?:[\\dA-F]{2} ){1,8}|(?:[\\dA-F]{4} )*[\\dA-F]{4}|(?:[\\dA-F]{2} )*[\\dA-F]{2})[^\\n]*\\n?){2,}$",
                flags: "i",
                args: []
            },
@@ -37,12 +37,12 @@ class FromMorseCode extends Operation {
                "value": WORD_DELIM_OPTIONS
            }
        ];
-        this.patterns = [
+        this.checks = [
            {
-                match: "(?:^[-. \\n]{5,}$|^[_. \\n]{5,}$|^(?:dash|dot| |\\n){5,}$)",
+                pattern: "(?:^[-. \\n]{5,}$|^[_. \\n]{5,}$|^(?:dash|dot| |\\n){5,}$)",
                flags: "i",
                args: ["Space", "Line feed"]
-            },
+            }
        ];
    }

@@ -32,37 +32,37 @@ class FromOctal extends Operation {
                "value": DELIM_OPTIONS
            }
        ];
-        this.patterns = [
+        this.checks = [
            {
-                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?: (?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                pattern: "^(?:[0-7]{1,2}|[123][0-7]{2})(?: (?:[0-7]{1,2}|[123][0-7]{2}))*$",
                flags: "",
                args: ["Space"]
            },
            {
-                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:,(?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                pattern: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:,(?:[0-7]{1,2}|[123][0-7]{2}))*$",
                flags: "",
                args: ["Comma"]
            },
            {
-                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:;(?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                pattern: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:;(?:[0-7]{1,2}|[123][0-7]{2}))*$",
                flags: "",
                args: ["Semi-colon"]
            },
            {
-                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?::(?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                pattern: "^(?:[0-7]{1,2}|[123][0-7]{2})(?::(?:[0-7]{1,2}|[123][0-7]{2}))*$",
                flags: "",
                args: ["Colon"]
            },
            {
-                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                pattern: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$",
                flags: "",
                args: ["Line feed"]
            },
            {
-                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\r\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                pattern: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\r\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$",
                flags: "",
                args: ["CRLF"]
-            },
+            }
        ];
    }

@@ -28,9 +28,9 @@ class FromQuotedPrintable extends Operation {
        this.inputType = "string";
        this.outputType = "byteArray";
        this.args = [];
-        this.patterns = [
+        this.checks = [
            {
-                match: "^[\\x21-\\x3d\\x3f-\\x7e \\t]{0,76}(?:=[\\da-f]{2}|=\\r?\\n)(?:[\\x21-\\x3d\\x3f-\\x7e \\t]|=[\\da-f]{2}|=\\r?\\n)*$",
+                pattern: "^[\\x21-\\x3d\\x3f-\\x7e \\t]{0,76}(?:=[\\da-f]{2}|=\\r?\\n)(?:[\\x21-\\x3d\\x3f-\\x7e \\t]|=[\\da-f]{2}|=\\r?\\n)*$",
                flags: "i",
                args: []
            },
@@ -33,27 +33,27 @@ class FromUNIXTimestamp extends Operation {
                "value": UNITS
            }
        ];
-        this.patterns = [
+        this.checks = [
            {
-                match: "^1?\\d{9}$",
+                pattern: "^1?\\d{9}$",
                flags: "",
                args: ["Seconds (s)"]
            },
            {
-                match: "^1?\\d{12}$",
+                pattern: "^1?\\d{12}$",
                flags: "",
                args: ["Milliseconds (ms)"]
            },
            {
-                match: "^1?\\d{15}$",
+                pattern: "^1?\\d{15}$",
                flags: "",
                args: ["Microseconds (μs)"]
            },
            {
-                match: "^1?\\d{18}$",
+                pattern: "^1?\\d{18}$",
                flags: "",
                args: ["Nanoseconds (ns)"]
-            },
+            }
        ];
    }

@@ -0,0 +1,184 @@
+/**
+ * @author pointhi [thomas.pointhuber@gmx.at]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation.mjs";
+import OperationError from "../errors/OperationError.mjs";
+import Utils from "../Utils.mjs";
+import {isImage} from "../lib/FileType";
+import {toBase64} from "../lib/Base64";
+import jimp from "jimp";
+import {isWorkerEnvironment} from "../Utils";
+
+/**
+ * Generate Image operation
+ */
+class GenerateImage extends Operation {
+
+    /**
+     * GenerateImage constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Generate Image";
+        this.module = "Image";
+        this.description = "Generates an image using the input as pixel values.";
+        this.infoURL = "";
+        this.inputType = "ArrayBuffer";
+        this.outputType = "ArrayBuffer";
+        this.presentType = "html";
+        this.args = [
+            {
+                "name": "Mode",
+                "type": "option",
+                "value": ["Greyscale", "RG", "RGB", "RGBA", "Bits"]
+            },
+            {
+                "name": "Pixel Scale Factor",
+                "type": "number",
+                "value": 8,
+            },
+            {
+                "name": "Pixels per row",
+                "type": "number",
+                "value": 64,
+            }
+        ];
+    }
+
+    /**
+     * @param {byteArray} input
+     * @param {Object[]} args
+     * @returns {ArrayBuffer}
+     */
+    async run(input, args) {
+        const [mode, scale, width] = args;
+        input = new Uint8Array(input);
+
+        if (scale <= 0) {
+            throw new OperationError("Pixel Scale Factor needs to be > 0");
+        }
+
+        if (width <= 0) {
+            throw new OperationError("Pixels per Row needs to be > 0");
+        }
+
+        const bytePerPixelMap = {
+            "Greyscale": 1,
+            "RG": 2,
+            "RGB": 3,
+            "RGBA": 4,
+            "Bits": 1/8,
+        };
+
+        const bytesPerPixel = bytePerPixelMap[mode];
+
+        if (bytesPerPixel > 0 && input.length % bytesPerPixel  !== 0) {
+            throw new OperationError(`Number of bytes is not a divisor of ${bytesPerPixel}`);
+        }
+
+        const height = Math.ceil(input.length / bytesPerPixel / width);
+        const image = await new jimp(width, height, (err, image) => {});
+
+        if (isWorkerEnvironment())
+            self.sendStatusMessage("Generating image from data...");
+
+        if (mode === "Bits") {
+            let index = 0;
+            for (let j = 0; j < input.length; j++) {
+                const curByte = Utils.bin(input[j]);
+                for (let k = 0; k < 8; k++, index++) {
+                    const x = index % width;
+                    const y = Math.floor(index / width);
+
+                    const value = curByte[k] === "0" ? 0xFF : 0x00;
+                    const pixel = jimp.rgbaToInt(value, value, value, 0xFF);
+                    image.setPixelColor(pixel, x, y);
+                }
+            }
+        } else {
+            let i = 0;
+            while (i < input.length) {
+                const index = i / bytesPerPixel;
+                const x = index % width;
+                const y = Math.floor(index / width);
+
+                let red = 0x00;
+                let green = 0x00;
+                let blue = 0x00;
+                let alpha = 0xFF;
+
+                switch (mode) {
+                    case "Greyscale":
+                        red = green = blue = input[i++];
+                        break;
+
+                    case "RG":
+                        red = input[i++];
+                        green = input[i++];
+                        break;
+
+                    case "RGB":
+                        red = input[i++];
+                        green = input[i++];
+                        blue = input[i++];
+                        break;
+
+                    case "RGBA":
+                        red = input[i++];
+                        green = input[i++];
+                        blue = input[i++];
+                        alpha = input[i++];
+                        break;
+
+                    default:
+                        throw new OperationError(`Unsupported Mode: (${mode})`);
+                }
+
+                try {
+                    const pixel = jimp.rgbaToInt(red, green, blue, alpha);
+                    image.setPixelColor(pixel, x, y);
+                } catch (err) {
+                    throw new OperationError(`Error while generating image from pixel values. (${err})`);
+                }
+            }
+        }
+
+        if (scale !== 1) {
+            if (isWorkerEnvironment())
+                self.sendStatusMessage("Scaling image...");
+
+            image.scaleToFit(width*scale, height*scale, jimp.RESIZE_NEAREST_NEIGHBOR);
+        }
+
+        try {
+            const imageBuffer = await image.getBufferAsync(jimp.MIME_PNG);
+            return imageBuffer.buffer;
+        } catch (err) {
+            throw new OperationError(`Error generating image. (${err})`);
+        }
+    }
+
+    /**
+     * Displays the generated image using HTML for web apps
+     * @param {ArrayBuffer} data
+     * @returns {html}
+     */
+    present(data) {
+        if (!data.byteLength) return "";
+        const dataArray = new Uint8Array(data);
+
+        const type = isImage(dataArray);
+        if (!type) {
+            throw new OperationError("Invalid file type.");
+        }
+
+        return `<img src="data:${type};base64,${toBase64(dataArray)}">`;
+    }
+
+}
+
+export default GenerateImage;
@@ -27,12 +27,12 @@ class Gunzip extends Operation {
        this.inputType = "ArrayBuffer";
        this.outputType = "ArrayBuffer";
        this.args = [];
-        this.patterns = [
+        this.checks = [
            {
-                match: "^\\x1f\\x8b\\x08",
+                pattern: "^\\x1f\\x8b\\x08",
                flags: "",
                args: []
-            },
+            }
        ];
    }

@@ -293,8 +293,8 @@ class Lorenz extends Operation {
            chosenSetting.S[3] = this.readLugs(lugs3);
            chosenSetting.S[4] = this.readLugs(lugs4);
            chosenSetting.S[5] = this.readLugs(lugs5);
-            chosenSetting.M[1] = this.readLugs(lugm37);
-            chosenSetting.M[2] = this.readLugs(lugm61);
+            chosenSetting.M[1] = this.readLugs(lugm61);
+            chosenSetting.M[2] = this.readLugs(lugm37);
            chosenSetting.X[1] = this.readLugs(lugx1);
            chosenSetting.X[2] = this.readLugs(lugx2);
            chosenSetting.X[3] = this.readLugs(lugx3);
@@ -23,19 +23,19 @@ class LuhnChecksum extends Operation {
        this.description = "The Luhn algorithm, also known as the modulus 10 or mod 10 algorithm, is a simple checksum formula used to validate a variety of identification numbers, such as credit card numbers, IMEI numbers and Canadian Social Insurance Numbers.";
        this.infoURL = "https://wikipedia.org/wiki/Luhn_algorithm";
        this.inputType = "string";
-        this.outputType = "number";
+        this.outputType = "string";
        this.args = [];
    }

    /**
-     * @param {string} input
-     * @param {Object[]} args
+     * Generates the Luhn Checksum from the input.
+     *
+     * @param {string} inputStr
     * @returns {number}
     */
-    run(input, args) {
+    checksum(inputStr) {
        let even = false;
-        return input.split("").reverse().reduce((acc, elem) => {
-
+        return inputStr.split("").reverse().reduce((acc, elem) => {
            // Convert element to integer.
            let temp = parseInt(elem, 10);

@@ -45,7 +45,6 @@ class LuhnChecksum extends Operation {

            // If element is in an even position
            if (even) {
-
                // Double the element and add the quotient and remainder together.
                temp = 2 * elem;
                temp = Math.floor(temp/10) + (temp % 10);
@@ -53,10 +52,26 @@ class LuhnChecksum extends Operation {

            even = !even;
            return acc + temp;
-
        }, 0)  % 10;
    }

+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        if (!input) return "";
+
+        const checkSum = this.checksum(input);
+        let checkDigit = this.checksum(input + "0");
+        checkDigit = checkDigit === 0 ? 0 : (10-checkDigit);
+
+        return `Checksum: ${checkSum}
+Checkdigit: ${checkDigit}
+Luhn Validated String: ${input + "" + checkDigit}`;
+    }
+
 }

 export default LuhnChecksum;
@@ -144,7 +144,7 @@ class MultipleBombe extends Operation {
     * @param {number} progress - Progress (as a float in the range 0..1)
     */
    updateStatus(nLoops, nStops, progress, start) {
-        const elapsed = new Date().getTime() - start;
+        const elapsed = Date.now() - start;
        const remaining = (elapsed / progress) * (1 - progress) / 1000;
        const hours = Math.floor(remaining / 3600);
        const minutes = `0${Math.floor((remaining % 3600) / 60)}`.slice(-2);
@@ -237,7 +237,7 @@ class MultipleBombe extends Operation {
        const totalRuns = choose(rotors.length, 3) * 6 * fourthRotors.length * reflectors.length;
        let nRuns = 0;
        let nStops = 0;
-        const start = new Date().getTime();
+        const start = Date.now();
        for (const rotor1 of rotors) {
            for (const rotor2 of rotors) {
                if (rotor2 === rotor1) {
@@ -33,9 +33,9 @@ class ParseQRCode extends Operation {
                "value": false
            }
        ];
-        this.patterns = [
+        this.checks = [
            {
-                "match": "^(?:\\xff\\xd8\\xff|\\x89\\x50\\x4e\\x47|\\x47\\x49\\x46|.{8}\\x57\\x45\\x42\\x50|\\x42\\x4d)",
+                "pattern": "^(?:\\xff\\xd8\\xff|\\x89\\x50\\x4e\\x47|\\x47\\x49\\x46|.{8}\\x57\\x45\\x42\\x50|\\x42\\x4d)",
                "flags": "",
                "args": [false],
                "useful": true
@@ -38,6 +38,13 @@ class ParseSSHHostKey extends Operation {
                ]
            }
        ];
+        this.checks = [
+            {
+                pattern:  "^\\s*([A-F\\d]{2}[,;:]){15,}[A-F\\d]{2}\\s*$",
+                flags:  "i",
+                args:   ["Hex"]
+            }
+        ];
    }

    /**
@@ -25,6 +25,13 @@ class ParseUNIXFilePermissions extends Operation {
        this.inputType = "string";
        this.outputType = "string";
        this.args = [];
+        this.checks = [
+            {
+                pattern:  "^\\s*d[rxw-]{9}\\s*$",
+                flags:  "",
+                args:   []
+            }
+        ];
    }

    /**
@@ -25,6 +25,13 @@ class ParseUserAgent extends Operation {
        this.inputType = "string";
        this.outputType = "string";
        this.args = [];
+        this.checks = [
+            {
+                pattern:  "^(User-Agent:|Mozilla\\/)[^\\n\\r]+\\s*$",
+                flags:  "i",
+                args:   []
+            }
+        ];
    }

    /**
@@ -35,13 +35,11 @@ class ParseX509Certificate extends Operation {
                "value": ["PEM", "DER Hex", "Base64", "Raw"]
            }
        ];
-        this.patterns = [
+        this.checks = [
            {
-                "match": "^-+BEGIN CERTIFICATE-+\\r?\\n[\\da-z+/\\n\\r]+-+END CERTIFICATE-+\\r?\\n?$",
+                "pattern": "^-+BEGIN CERTIFICATE-+\\r?\\n[\\da-z+/\\n\\r]+-+END CERTIFICATE-+\\r?\\n?$",
                "flags": "i",
-                "args": [
-                    "PEM"
-                ]
+                "args": ["PEM"]
            }
        ];
    }
@@ -60,6 +60,12 @@ class RawInflate extends Operation {
                value: false
            }
        ];
+        this.checks = [
+            {
+                entropyRange: [7.5, 8],
+                args: [0, 0, INFLATE_BUFFER_TYPE, false, false]
+            }
+        ];
    }

    /**
@@ -163,7 +163,7 @@ class RegularExpression extends Operation {
                    case "List matches with capture groups":
                        return Utils.escapeHtml(regexList(input, regex, displayTotal, true, true));
                    default:
-                        return "Error: Invalid output format";
+                        throw new OperationError("Error: Invalid output format");
                }
            } catch (err) {
                throw new OperationError("Invalid regex. Details: " + err.message);
@@ -35,12 +35,15 @@ class RenderImage extends Operation {
                "value": ["Raw", "Base64", "Hex"]
            }
        ];
-        this.patterns = [
+        this.checks = [
            {
-                "match": "^(?:\\xff\\xd8\\xff|\\x89\\x50\\x4e\\x47|\\x47\\x49\\x46|.{8}\\x57\\x45\\x42\\x50|\\x42\\x4d)",
-                "flags": "",
-                "args": ["Raw"],
-                "useful": true
+                pattern: "^(?:\\xff\\xd8\\xff|\\x89\\x50\\x4e\\x47|\\x47\\x49\\x46|.{8}\\x57\\x45\\x42\\x50|\\x42\\x4d)",
+                flags: "",
+                args: ["Raw"],
+                useful: true,
+                output: {
+                    mime: "image"
+                }
            }
        ];
    }
@@ -35,6 +35,13 @@ class StripHTMLTags extends Operation {
                "value": true
            }
        ];
+        this.checks = [
+            {
+                pattern:  "(</html>|</div>|</body>)",
+                flags:  "i",
+                args:   [true, true]
+            }
+        ];
    }

    /**
@@ -24,6 +24,13 @@ class StripHTTPHeaders extends Operation {
        this.inputType = "string";
        this.outputType = "string";
        this.args = [];
+        this.checks = [
+            {
+                pattern:  "^HTTP(.|\\s)+?(\\r?\\n){2}",
+                flags:  "",
+                args:   []
+            }
+        ];
    }

    /**
@@ -24,9 +24,9 @@ class URLDecode extends Operation {
        this.inputType = "string";
        this.outputType = "string";
        this.args = [];
-        this.patterns = [
+        this.checks = [
            {
-                match: ".*(?:%[\\da-f]{2}.*){4}",
+                pattern: ".*(?:%[\\da-f]{2}.*){4}",
                flags: "i",
                args: []
            },
@@ -27,9 +27,9 @@ class Untar extends Operation {
        this.outputType = "List<File>";
        this.presentType = "html";
        this.args = [];
-        this.patterns = [
+        this.checks = [
            {
-                "match": "^.{257}\\x75\\x73\\x74\\x61\\x72",
+                "pattern": "^.{257}\\x75\\x73\\x74\\x61\\x72",
                "flags": "",
                "args": []
            }
@@ -40,12 +40,12 @@ class Unzip extends Operation {
                value: false
            }
        ];
-        this.patterns = [
+        this.checks = [
            {
-                match: "^\\x50\\x4b(?:\\x03|\\x05|\\x07)(?:\\x04|\\x06|\\x08)",
+                pattern: "^\\x50\\x4b(?:\\x03|\\x05|\\x07)(?:\\x04|\\x06|\\x08)",
                flags: "",
                args: ["", false]
-            },
+            }
        ];
    }

@@ -59,9 +59,9 @@ class ZlibInflate extends Operation {
                value: false
            }
        ];
-        this.patterns = [
+        this.checks = [
            {
-                match: "^\\x78(\\x01|\\x9c|\\xda|\\x5e)",
+                pattern: "^\\x78(\\x01|\\x9c|\\xda|\\x5e)",
                flags: "",
                args: [0, 0, "Adaptive", false, false]
            },
@@ -70,7 +70,7 @@ if (typeof document !== 'undefined') {
    try {
        // Mouse move event to fill random array
        document.addEventListener('mousemove', function (e) {
-            randomRing.set((new Date().getTime() & 255) ^
+            randomRing.set((Date.now() & 255) ^
                    ((e.clientX || e.pageX) & 255) ^
                    ((e.clientY || e.pageY) & 255));
        }, false);
@@ -80,7 +80,7 @@ if (typeof document !== 'undefined') {
    try {
        // Keypress event to fill random array
        document.addEventListener('keydown', function (e) {
-            randomRing.set((new Date().getTime() & 255) ^
+            randomRing.set((Date.now() & 255) ^
                    (e.keyCode & 255));
        }, false);
    } catch (e) {
@@ -282,11 +282,11 @@ export function help(input) {
        .map(result => result.hydrated);

    if (matches && matches.length) {
-        console.log(`${matches.length} result${matches.length > 1 ? "s" : ""} found.`);
+        // console.log(`${matches.length} result${matches.length > 1 ? "s" : ""} found.`);
        return matches;
    }

-    console.log("No results found.");
+    // console.log("No results found.");
    return null;
 }

@@ -1,4 +1,4 @@
-import Sitemap from "sitemap";
+import sm from "sitemap";
 import OperationConfig from "../../core/config/OperationConfig.json";


@@ -10,24 +10,25 @@ import OperationConfig from "../../core/config/OperationConfig.json";
 * @license Apache-2.0
 */

-const sitemap = Sitemap.createSitemap({
+const smStream = new sm.SitemapStream({
    hostname: "https://gchq.github.io/CyberChef",
 });

-sitemap.add({
+smStream.write({
    url: "/",
    changefreq: "weekly",
    priority: 1.0
 });

 for (const op in OperationConfig) {
-    sitemap.add({
+    smStream.write({
        url: `/?op=${encodeURIComponent(op)}`,
        changeFreq: "yearly",
        priority: 0.5
    });
 }
+smStream.end();

-const xml = sitemap.toString();
-
-console.log(xml); // eslint-disable-line no-console
+sm.streamToPromise(smStream).then(
+    buffer => console.log(buffer.toString()) // eslint-disable-line no-console
+);
@@ -28,6 +28,10 @@ class SeasonalWaiter {
        // Konami code
        this.kkeys = [];
        window.addEventListener("keydown", this.konamiCodeListener.bind(this));
+
+        // CyberChef Challenge
+        // eslint-disable-next-line no-console
+        console.log("43 6f 6e 67 72 61 74 75 6c 61 74 69 6f 6e 73 2c 20 79 6f 75 20 68 61 76 65 20 63 6f 6d 70 6c 65 74 65 64 20 43 79 62 65 72 43 68 65 66 20 63 68 61 6c 6c 65 6e 67 65 20 23 31 21 0a 0a 54 68 69 73 20 63 68 61 6c 6c 65 6e 67 65 20 65 78 70 6c 6f 72 65 64 20 68 65 78 61 64 65 63 69 6d 61 6c 20 65 6e 63 6f 64 69 6e 67 2e 20 54 6f 20 6c 65 61 72 6e 20 6d 6f 72 65 2c 20 76 69 73 69 74 20 77 69 6b 69 70 65 64 69 61 2e 6f 72 67 2f 77 69 6b 69 2f 48 65 78 61 64 65 63 69 6d 61 6c 2e 0a 0a 54 68 65 20 63 6f 64 65 20 66 6f 72 20 74 68 69 73 20 63 68 61 6c 6c 65 6e 67 65 20 69 73 20 39 64 34 63 62 63 65 66 2d 62 65 35 32 2d 34 37 35 31 2d 61 32 62 32 2d 38 33 33 38 65 36 34 30 39 34 31 36 20 28 6b 65 65 70 20 74 68 69 73 20 70 72 69 76 61 74 65 29 2e 0a 0a 54 68 65 20 6e 65 78 74 20 63 68 61 6c 6c 65 6e 67 65 20 63 61 6e 20 62 65 20 66 6f 75 6e 64 20 61 74 20 68 74 74 70 73 3a 2f 2f 70 61 73 74 65 62 69 6e 2e 63 6f 6d 2f 47 53 6e 54 41 6d 6b 56 2e");
    }


@@ -37,7 +37,7 @@ class WindowWaiter {
     * focus is returned.
     */
    windowBlur() {
-        this.windowBlurTime = new Date().getTime();
+        this.windowBlurTime = Date.now();
    }


@@ -52,7 +52,7 @@ class WindowWaiter {
     * a long time and the browser has swapped out all its memory.
     */
    windowFocus() {
-        const unfocusedTime = new Date().getTime() - this.windowBlurTime;
+        const unfocusedTime = Date.now() - this.windowBlurTime;
        if (unfocusedTime > 60000) {
            this.app.silentBake();
        }
@@ -375,7 +375,7 @@ class WorkerWaiter {
     */
    bakingComplete() {
        this.setBakingStatus(false);
-        let duration = new Date().getTime() - this.bakeStartTime;
+        let duration = Date.now() - this.bakeStartTime;
        duration = duration.toLocaleString() + "ms";
        const progress = this.getBakeProgress();

@@ -489,7 +489,7 @@ class WorkerWaiter {
    bake(recipeConfig, options, progress, step) {
        this.setBakingStatus(true);
        this.manager.recipe.updateBreakpointIndicator(false);
-        this.bakeStartTime = new Date().getTime();
+        this.bakeStartTime = Date.now();
        this.bakeId++;
        this.recipeConfig = recipeConfig;
        this.options = options;
@@ -10,6 +10,8 @@
 * @license Apache-2.0
 */
 import Chef from "../../src/core/Chef.mjs";
+import Utils from "../../src/core/Utils.mjs";
+import cliProgress from "cli-progress";

 /**
 * Object to store and run the list of tests.
@@ -47,68 +49,103 @@ class TestRegister {
    /**
     * Runs all the tests in the register.
     */
-    runTests () {
-        console.log("Running tests...");
-        return Promise.all(
-            this.tests.map(function(test, i) {
-                const chef = new Chef();
+    async runTests () {
+        const progBar = new cliProgress.SingleBar({
+            format: formatter,
+            stopOnComplete: true
+        }, cliProgress.Presets.shades_classic);
+        const testResults = [];

-                return chef.bake(
-                    test.input,
-                    test.recipeConfig,
-                    {},
-                    0,
-                    false
-                ).then(function(result) {
-                    const ret = {
-                        test: test,
-                        status: null,
-                        output: null,
-                    };
+        console.log("Running operation tests...");
+        progBar.start(this.tests.length, 0, {
+            msg: "Setting up"
+        });

-                    if (result.error) {
-                        if (test.expectedError) {
-                            ret.status = "passing";
-                        } else {
-                            ret.status = "erroring";
-                            ret.output = result.error.displayStr;
-                        }
-                    } else {
-                        if (test.expectedError) {
-                            ret.status = "failing";
-                            ret.output = "Expected an error but did not receive one.";
-                        } else if (result.result === test.expectedOutput) {
-                            ret.status = "passing";
-                        } else if ("expectedMatch" in test && test.expectedMatch.test(result.result)) {
-                            ret.status = "passing";
-                        } else {
-                            ret.status = "failing";
-                            const expected = test.expectedOutput ? test.expectedOutput :
-                                test.expectedMatch ? test.expectedMatch.toString() : "unknown";
-                            ret.output = [
-                                "Expected",
-                                "\t" + expected.replace(/\n/g, "\n\t"),
-                                "Received",
-                                "\t" + result.result.replace(/\n/g, "\n\t"),
-                            ].join("\n");
-                        }
-                    }
+        for (const test of this.tests) {
+            progBar.update(testResults.length, {
+                msg: test.name
+            });

-                    return ret;
-                });
-            })
-        );
+            const chef = new Chef();
+            const result = await chef.bake(
+                test.input,
+                test.recipeConfig,
+                {},
+                0,
+                false
+            );
+
+            const ret = {
+                test: test,
+                status: null,
+                output: null,
+                duration: result.duration
+            };
+
+            if (result.error) {
+                if (test.expectedError) {
+                    ret.status = "passing";
+                } else {
+                    ret.status = "erroring";
+                    ret.output = result.error.displayStr;
+                }
+            } else {
+                if (test.expectedError) {
+                    ret.status = "failing";
+                    ret.output = "Expected an error but did not receive one.";
+                } else if (result.result === test.expectedOutput) {
+                    ret.status = "passing";
+                } else if ("expectedMatch" in test && test.expectedMatch.test(result.result)) {
+                    ret.status = "passing";
+                } else if ("unexpectedMatch" in test && !test.unexpectedMatch.test(result.result)) {
+                    ret.status = "passing";
+                } else {
+                    ret.status = "failing";
+                    const expected = test.expectedOutput ? test.expectedOutput :
+                        test.expectedMatch ? test.expectedMatch.toString() :
+                            test.unexpectedMatch ? "to not find " + test.unexpectedMatch.toString() :
+                                "unknown";
+                    ret.output = [
+                        "Expected",
+                        "\t" + expected.replace(/\n/g, "\n\t"),
+                        "Received",
+                        "\t" + result.result.replace(/\n/g, "\n\t"),
+                    ].join("\n");
+                }
+            }
+
+            testResults.push(ret);
+            progBar.increment();
+        }
+
+        return testResults;
    }

    /**
     * Run all api related tests and wrap results in report format
     */
-    runApiTests() {
-        return Promise.all(this.apiTests.map(async function(test, i) {
+    async runApiTests() {
+        const progBar = new cliProgress.SingleBar({
+            format: formatter,
+            stopOnComplete: true
+        }, cliProgress.Presets.shades_classic);
+        const testResults = [];
+
+        console.log("Running Node API tests...");
+        progBar.start(this.apiTests.length, 0, {
+            msg: "Setting up"
+        });
+
+        global.TESTING = true;
+        for (const test of this.apiTests) {
+            progBar.update(testResults.length, {
+                msg: test.name
+            });
+
            const result = {
                test: test,
                status: null,
-                output: null,
+                output: null
            };
            try {
                await test.run();
@@ -117,10 +154,37 @@ class TestRegister {
                result.status = "erroring";
                result.output = e.message;
            }
-            return result;
-        }));
+
+            testResults.push(result);
+            progBar.increment();
+        }
+
+        return testResults;
    }
 }

+
+/**
+ * Formatter for the progress bar
+ *
+ * @param {Object} options
+ * @param {Object} params
+ * @param {Object} payload
+ * @returns {string}
+ */
+function formatter(options, params, payload) {
+    const bar = options.barCompleteString.substr(0, Math.round(params.progress * options.barsize)) +
+        options.barIncompleteString.substr(0, Math.round((1-params.progress) * options.barsize));
+
+    const percentage = Math.floor(params.progress * 100),
+        duration = Math.floor((Date.now() - params.startTime) / 1000);
+
+    let testName = payload.msg ? payload.msg : "";
+    if (params.value >= params.total) testName = "Tests completed";
+    testName = Utils.truncate(testName, 25).padEnd(25, " ");
+
+    return `${testName} ${bar} ${params.value}/${params.total} | ${percentage}% | Duration: ${duration}s`;
+}
+
 // Export an instance to make a singleton
 export default new TestRegister();
@@ -33,6 +33,10 @@ function handleTestResult(testStatus, testResult) {
    testStatus.allTestsPassing = testStatus.allTestsPassing && testResult.status === "passing";
    testStatus.counts[testResult.status] = (testStatus.counts[testResult.status] || 0) + 1;
    testStatus.counts.total += 1;
+
+    if (testResult.duration > 2000) {
+        console.log(`'${testResult.test.name}' took ${(testResult.duration / 1000).toFixed(1)}s to complete`);
+    }
 }

 /**
@@ -42,8 +46,6 @@ function handleTestResult(testStatus, testResult) {
 * @param {Object[]} results - results from TestRegister
 */
 export function logTestReport(testStatus, results) {
-    console.log("Tests completed.");
-
    results.forEach(r => handleTestResult(testStatus, r));

    console.log();
@@ -80,8 +82,9 @@ export function logTestReport(testStatus, results) {
 * Fail if the process takes longer than 60 seconds.
 */
 export function setLongTestFailure() {
+    const timeLimit = 120;
    setTimeout(function() {
-        console.log("Tests took longer than 60 seconds to run, returning.");
+        console.log(`Tests took longer than ${timeLimit} seconds to run, returning.`);
        process.exit(1);
-    }, 60 * 1000);
+    }, timeLimit * 1000);
 }
@@ -35,6 +35,7 @@ setLongTestFailure();

 const logOpsTestReport = logTestReport.bind(null, testStatus);

-TestRegister.runApiTests()
-    .then(logOpsTestReport);
-
+(async function() {
+    const results = await TestRegister.runApiTests();
+    logOpsTestReport(results);
+})();
@@ -588,7 +588,7 @@ Password: 034148`;
        const result = await chef.generatePGPKeyPair("Back To the Drawing Board", {
            keyType: "ECC-256",
        });
-        assert.strictEqual(result.toString().length, 2005);
+        assert.strictEqual(result.toString().length, 2007);
    }),

    it("Generate UUID", () => {
@@ -116,5 +116,8 @@ setLongTestFailure();

 const logOpsTestReport = logTestReport.bind(null, testStatus);

-TestRegister.runTests()
-    .then(logOpsTestReport);
+(async function() {
+    const results = await TestRegister.runTests();
+    logOpsTestReport(results);
+})();
+
@@ -0,0 +1,32 @@
+/**
+ * @author MarvinJWendt [git@marvinjwendt.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import TestRegister from "../../lib/TestRegister.mjs";
+
+TestRegister.addTests([
+    {
+        name: "Convert to NATO alphabet: nothing",
+        input: "",
+        expectedOutput: "",
+        recipeConfig: [
+            {
+                op: "Convert to NATO alphabet",
+                args: []
+            }
+        ]
+    },
+    {
+        name: "Convert to NATO alphabet: full alphabet with numbers",
+        input: "abcdefghijklmnopqrstuvwxyz0123456789,/.",
+        expectedOutput: "Alfa Bravo Charlie Delta Echo Foxtrot Golf Hotel India Juliett Kilo Lima Mike November Oscar Papa Quebec Romeo Sierra Tango Uniform Victor Whiskey X-ray Yankee Zulu Zero One Two Three Four Five Six Seven Eight Nine Comma Fraction bar Full stop ",
+        recipeConfig: [
+            {
+                op: "Convert to NATO alphabet",
+                args: []
+            }
+        ]
+    }
+]);
@@ -92,6 +92,19 @@ TestRegister.addTests([
                ]
            }
        ]
+    },
+    {
+        name: "0x with Comma to Ascii",
+        input: "0x74,0x65,0x73,0x74,0x20,0x73,0x74,0x72,0x69,0x6e,0x67",
+        expectedOutput: "test string",
+        recipeConfig: [
+            {
+                "op": "From Hex",
+                "args": [
+                    "0x with comma"
+                ]
+            }
+        ]

-    }
+    },
 ]);
@@ -11,7 +11,29 @@ TestRegister.addTests([
    {
        name: "Luhn Checksum on standard data",
        input: "35641709012469",
-        expectedOutput: "7",
+        expectedOutput: "Checksum: 7\nCheckdigit: 0\nLuhn Validated String: 356417090124690",
+        recipeConfig: [
+            {
+                op: "Luhn Checksum",
+                args: []
+            },
+        ],
+    },
+    {
+        name: "Luhn Checksum on standard data 2",
+        input: "896101950123440000",
+        expectedOutput: "Checksum: 5\nCheckdigit: 1\nLuhn Validated String: 8961019501234400001",
+        recipeConfig: [
+            {
+                op: "Luhn Checksum",
+                args: []
+            },
+        ],
+    },
+    {
+        name: "Luhn Checksum on standard data 3",
+        input: "35726908971331",
+        expectedOutput: "Checksum: 6\nCheckdigit: 7\nLuhn Validated String: 357269089713317",
        recipeConfig: [
            {
                op: "Luhn Checksum",
@@ -33,7 +55,7 @@ TestRegister.addTests([
    {
        name: "Luhn Checksum on empty data",
        input: "",
-        expectedOutput: "0",
+        expectedOutput: "",
        recipeConfig: [
            {
                op: "Luhn Checksum",
Author	SHA1	Message	Date
n1474335	0212bfb46e	9.19.0	2020-03-24 11:37:30 +00:00
n1474335	5e0d661542	Updated CHANGELOG	2020-03-24 11:37:26 +00:00
n1474335	4c5f529ef4	Merge branch 'n1073645-newMagic'	2020-03-24 11:07:20 +00:00
n1474335	b765534b8b	Tidied up the Magic operation	2020-03-24 11:06:37 +00:00
n1474335	26fa66ef64	Merge branch 'newMagic' of https://github.com/n1073645/CyberChef into n1073645-newMagic	2020-03-20 14:51:40 +00:00
n1474335	26b19350f2	9.18.2	2020-03-18 16:47:53 +00:00
n1474335	2018b7e247	Fixed sitemap	2020-03-18 16:47:48 +00:00
n1474335	cbff4161a1	9.18.1	2020-03-18 16:12:38 +00:00
n1474335	130bdfb7f2	Updated dependencies	2020-03-18 16:12:30 +00:00
n1474335	d0c43f5aa9	Added a challenge	2020-03-18 15:45:40 +00:00
n1474335	f864a5f31e	Suppressed highlighting errors	2020-03-18 13:40:16 +00:00
n1474335	cecae671d8	9.18.0	2020-03-13 17:13:19 +00:00
n1474335	b4e23ac454	Updated CHANGELOG	2020-03-13 17:13:12 +00:00
n1474335	e7209ca085	Merge branch 'MarvinJWendt-operation/convert-to-nato-alphabet'	2020-03-13 17:10:50 +00:00
n1474335	022ef71d2c	Tidied up 'Convert to NATO alphabet' operation	2020-03-13 17:10:29 +00:00
n1474335	0fad891a3a	Merge branch 'operation/convert-to-nato-alphabet' of https://github.com/MarvinJWendt/CyberChef into MarvinJWendt-operation/convert-to-nato-alphabet	2020-03-13 16:45:37 +00:00
n1474335	47f608b502	9.17.0	2020-03-13 16:38:03 +00:00
n1474335	4ab745f730	Updated CHANGELOG	2020-03-13 16:37:41 +00:00
n1474335	1f89ac11d2	Merge branch 'pointhi-generate_image'	2020-03-13 16:35:33 +00:00
n1474335	1a5dae76c2	Tidied up 'Generate Image' operation	2020-03-13 16:35:19 +00:00
n1474335	032c7f529a	Merge branch 'generate_image' of https://github.com/pointhi/CyberChef into pointhi-generate_image	2020-03-13 16:20:45 +00:00
n1474335	707818abcc	Merge branch 'n1073645-importFix'	2020-03-13 16:18:31 +00:00
n1474335	58e8b4c618	Merge branch 'importFix' of https://github.com/n1073645/CyberChef into n1073645-importFix	2020-03-13 16:18:10 +00:00
n1474335	9c0c2867dd	Increased test timeout to 120s from 60	2020-03-13 15:06:40 +00:00
n1474335	4308c717c3	Tests now display a progress bar and report long running tests	2020-03-13 14:59:48 +00:00
n1073645	342b67581b	Very small correction for import in Colossus	2020-03-13 10:14:08 +00:00
n1474335	75da5b650c	Replaced 'new Date().getTime()' calls with 'Date.now()' for clarity and performance	2020-03-12 15:23:22 +00:00
n1073645	5b6a53be3e	Docstrings added for Magic functions	2020-03-12 14:55:19 +00:00
n1073645	5b5105c864	Caching added for Magic regexes	2020-03-12 14:45:40 +00:00
n1474335	9d09146f68	9.16.4	2020-03-12 14:42:10 +00:00
n1474335	1b19d20d0c	Merge branch 'n1073645-Luhn'	2020-03-12 14:42:03 +00:00
n1474335	0eacab5ddc	Tidied up Luhn checksum op	2020-03-12 14:41:46 +00:00
n1474335	0d7874bac1	Merge branch 'Luhn' of https://github.com/n1073645/CyberChef into n1073645-Luhn	2020-03-12 14:35:40 +00:00
n1474335	5cddfafbd0	9.16.3	2020-03-12 14:31:49 +00:00
n1474335	5ce133c47e	Merge branch 'VirtualColossus-master'	2020-03-12 14:31:38 +00:00
n1073645	570a84b67a	More Magic tests	2020-03-11 16:27:37 +00:00
VirtualColossus	a68bfd7223	Fix gchq#973 custom setting correction	2020-03-11 13:01:49 +00:00
n1073645	fd7176a445	Extra Magic Tests	2020-03-11 12:51:46 +00:00
n1073645	0a06472639	Test added for From Hex	2020-03-10 11:23:14 +00:00
n1073645	3f3a7cd4f6	From Hex Regexes	2020-03-10 11:12:43 +00:00
n1073645	99415359d0	Extra Magic Tests	2020-03-10 09:39:13 +00:00
n1073645	54cb2d268b	Luhn checksum tests	2020-03-09 09:37:34 +00:00
n1073645	0e40daecb6	Generates both the checksum and checkdigit.	2020-03-09 09:13:02 +00:00
n1474335	21a822b082	9.16.2	2020-03-06 16:00:06 +00:00
n1474335	a770d09687	Merge branch 'n1073645-debExtractor'	2020-03-06 15:59:59 +00:00
n1474335	27b81c4e11	Tidied up JAR and DEB extractors	2020-03-06 15:59:42 +00:00
n1474335	8826c80e07	Merge branch 'debExtractor' of https://github.com/n1073645/CyberChef into n1073645-debExtractor	2020-03-06 15:57:16 +00:00
n1073645	02e3ce7fc1	Linting	2020-03-06 14:50:25 +00:00
n1073645	673e6aede5	Moved alternative JAR signature	2020-03-06 14:26:05 +00:00
n1073645	14190fc533	DEB extractor	2020-03-05 15:01:07 +00:00
n1073645	20d0ae5304	Linting corrections	2020-02-25 11:35:39 +00:00
n1073645	2ba37af109	extra signatures	2020-02-25 11:33:35 +00:00
n1073645	728f8e65d6	Magic rebuild	2020-02-25 11:27:03 +00:00
Thomas Pointhuber	ef61735f64	Fix typo	2019-10-12 17:52:16 +02:00
Thomas Pointhuber	a2780ca056	Add bitwse mode to Generate Image operation	2019-10-12 17:35:46 +02:00
Thomas Pointhuber	d025c8bd9a	Add new operation to generate image from raw data	2019-10-12 17:13:14 +02:00
Marvin Wendt	acf38e47ba	Add ConvertToNATOAlphabet tests	2019-10-11 15:32:14 +02:00
Marvin Wendt	4122d4207d	Add ConvertToNATOAlphabet	2019-10-11 15:32:06 +02:00
Marvin Wendt	d550ae7d93	Add operation to categories	2019-10-11 15:31:46 +02:00
Marvin Wendt	815a542cc1	package-lock automaitcally updated on install	2019-10-11 15:31:25 +02:00