8.24.0

Negative numbers shouldn't make it go weird any more.
Automatic detection of input formats should be more reliable.

CHANGELOG.md

@@ -2,6 +2,15 @@
 All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master).
 
 
+### [8.24.0] - 2019-02-08
+- 'DNS over HTTPS' operation added [@h345983745] | [#489]
+
+### [8.23.1] - 2019-01-18
+- 'Convert co-ordinate format' operation added [@j433866] | [#476]
+
+### [8.23.0] - 2019-01-18
+- 'YARA Rules' operation added [@artemisbot] | [#468]
+
 ### [8.22.0] - 2019-01-10
 - 'Subsection' operation added [@j433866] | [#467]
 
@@ -97,6 +106,9 @@ All major and minor version changes will be documented in this file. Details of
 
 
 
+[8.24.0]: https://github.com/gchq/CyberChef/releases/tag/v8.24.0
+[8.23.1]: https://github.com/gchq/CyberChef/releases/tag/v8.23.1
+[8.23.0]: https://github.com/gchq/CyberChef/releases/tag/v8.23.0
 [8.22.0]: https://github.com/gchq/CyberChef/releases/tag/v8.22.0
 [8.21.0]: https://github.com/gchq/CyberChef/releases/tag/v8.21.0
 [8.20.0]: https://github.com/gchq/CyberChef/releases/tag/v8.20.0
@@ -129,6 +141,7 @@ All major and minor version changes will be documented in this file. Details of
 [@d98762625]: https://github.com/d98762625
 [@j433866]: https://github.com/j433866
 [@GCHQ77703]: https://github.com/GCHQ77703
+[@h345983745]: https://github.com/h345983745
 [@artemisbot]: https://github.com/artemisbot
 [@picapi]: https://github.com/picapi
 [@Dachande663]: https://github.com/Dachande663
@@ -176,3 +189,6 @@ All major and minor version changes will be documented in this file. Details of
 [#458]: https://github.com/gchq/CyberChef/pull/458
 [#461]: https://github.com/gchq/CyberChef/pull/461
 [#467]: https://github.com/gchq/CyberChef/pull/467
+[#468]: https://github.com/gchq/CyberChef/pull/468
+[#476]: https://github.com/gchq/CyberChef/pull/476
+[#489]: https://github.com/gchq/CyberChef/pull/489

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "8.22.1",
+  "version": "8.24.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -766,6 +766,22 @@
         "semver": "^5.3.0"
       }
     },
+    "@babel/runtime-corejs2": {
+      "version": "7.3.1",
+      "resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.3.1.tgz",
+      "integrity": "sha512-YpO13776h3e6Wy8dl2J8T9Qwlvopr+b4trCEhHE+yek6yIqV8sx6g3KozdHMbXeBpjosbPi+Ii5Z7X9oXFHUKA==",
+      "requires": {
+        "core-js": "^2.5.7",
+        "regenerator-runtime": "^0.12.0"
+      },
+      "dependencies": {
+        "regenerator-runtime": {
+          "version": "0.12.1",
+          "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.12.1.tgz",
+          "integrity": "sha512-odxIc1/vDlo4iZcfXqRYFj0vpXFNoGdKMAUieAlFYO6m/nl5e9KR/beGf41z4a1FI+aQgtjhuaSlDxQ0hmkrHg=="
+        }
+      }
+    },
     "@babel/template": {
       "version": "7.2.2",
       "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.2.2.tgz",
@@ -2087,9 +2103,9 @@
       "dev": true
     },
     "bignumber.js": {
-      "version": "8.0.1",
-      "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-8.0.1.tgz",
-      "integrity": "sha512-zAySveTJXkgLYCBi0b14xzfnOs+f3G6x36I8w2a1+PFQpWk/dp0mI0F+ZZK2bu+3ELewDcSyP+Cfq++NcHX7sg=="
+      "version": "8.0.2",
+      "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-8.0.2.tgz",
+      "integrity": "sha512-EiuvFrnbv0jFixEQ9f58jo7X0qI2lNGIr/MxntmVzQc5JUweDSh8y8hbTCAomFtqwUPIOWcLXP0VEOSZTG7FFw=="
     },
     "binary-extensions": {
       "version": "1.12.0",
@@ -5098,14 +5114,12 @@
         "balanced-match": {
           "version": "1.0.0",
           "bundled": true,
-          "dev": true,
-          "optional": true
+          "dev": true
         },
         "brace-expansion": {
           "version": "1.1.11",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "balanced-match": "^1.0.0",
             "concat-map": "0.0.1"
@@ -5125,8 +5139,7 @@
         "concat-map": {
           "version": "0.0.1",
           "bundled": true,
-          "dev": true,
-          "optional": true
+          "dev": true
         },
         "console-control-strings": {
           "version": "1.1.0",
@@ -5274,7 +5287,6 @@
           "version": "3.0.4",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "brace-expansion": "^1.1.7"
           }
@@ -5684,6 +5696,11 @@
         "globule": "^1.0.0"
       }
     },
+    "geodesy": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/geodesy/-/geodesy-1.1.3.tgz",
+      "integrity": "sha512-H/0XSd1KjKZGZ2YGZcOYzRyY/foYAawwTEumNSo+YUwf+u5d4CfvBRg2i2Qimrx9yUEjWR8hLvMnhghuVFN0Zg=="
+    },
     "get-caller-file": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz",
@@ -7759,6 +7776,11 @@
       "resolved": "https://registry.npmjs.org/lex-parser/-/lex-parser-0.1.4.tgz",
       "integrity": "sha1-ZMTwJfF/1Tv7RXY/rrFvAVp0dVA="
     },
+    "libyara-wasm": {
+      "version": "0.0.11",
+      "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.11.tgz",
+      "integrity": "sha512-rglapPFo0IHPNksWYQXI8oqftXYj5mOGOf4BXtbSySVRX71pro4BehNjJ5qEpjYx+roGvNkcAD9zCsitA08sxw=="
+    },
     "livereload-js": {
       "version": "2.4.0",
       "resolved": "https://registry.npmjs.org/livereload-js/-/livereload-js-2.4.0.tgz",
@@ -9760,7 +9782,7 @@
       "dependencies": {
         "async": {
           "version": "1.5.2",
-          "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz",
+          "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz",
           "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=",
           "dev": true
         }
@@ -13825,9 +13847,12 @@
       "integrity": "sha512-fg03WRxtkCV6ohClePNAECYsmpKKTv5L8y/X3Dn1hQrec3POx2jHZ/0P2qQ6HvsrU1BmeqXcof3NGGueG6LxwQ=="
     },
     "xregexp": {
-      "version": "4.2.0",
-      "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-4.2.0.tgz",
-      "integrity": "sha512-IyMa7SVe9FyT4WbQVW3b95mTLVceHhLEezQ02+QMvmIqDnKTxk0MLWIQPSW2MXAr1zQb+9yvwYhcyQULneh3wA=="
+      "version": "4.2.4",
+      "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-4.2.4.tgz",
+      "integrity": "sha512-sO0bYdYeJAJBcJA8g7MJJX7UrOZIfJPd8U2SC7B2Dd/J24U0aQNoGp33shCaBSWeb0rD5rh6VBUIXOkGal1TZA==",
+      "requires": {
+        "@babel/runtime-corejs2": "^7.2.0"
+      }
     },
     "xtend": {
       "version": "4.0.1",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "8.22.1",
+  "version": "8.24.0",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",
@@ -80,7 +80,7 @@
     "babel-plugin-transform-builtin-extend": "1.1.2",
     "babel-polyfill": "^6.26.0",
     "bcryptjs": "^2.4.3",
-    "bignumber.js": "^8.0.1",
+    "bignumber.js": "^8.0.2",
     "bootstrap-colorpicker": "^2.5.3",
     "bootstrap-material-design": "^4.1.1",
     "bson": "^4.0.1",
@@ -95,6 +95,7 @@
     "esprima": "^4.0.1",
     "exif-parser": "^0.1.12",
     "file-saver": "^2.0.0",
+    "geodesy": "^1.1.3",
     "highlight.js": "^9.13.1",
     "jimp": "^0.6.0",
     "jquery": "^3.3.1",
@@ -106,6 +107,7 @@
     "jsqr": "^1.1.1",
     "jsrsasign": "8.0.12",
     "kbpgp": "^2.0.82",
+    "libyara-wasm": "0.0.11",
     "lodash": "^4.17.11",
     "loglevel": "^1.6.1",
     "loglevel-message-prefix": "^3.0.0",
@@ -129,7 +131,7 @@
     "vkbeautify": "^0.99.3",
     "xmldom": "^0.1.27",
     "xpath": "0.0.27",
-    "xregexp": "^4.2.0",
+    "xregexp": "^4.2.4",
     "zlibjs": "^0.3.1"
   },
   "scripts": {

src/core/Dish.mjs

@@ -168,7 +168,7 @@ class Dish {
                     this.value = Array.prototype.slice.call(new Uint8Array(this.value));
                     break;
                 case Dish.BIG_NUMBER:
-                    this.value = this.value instanceof BigNumber ? Utils.strToByteArray(this.value.toFixed()) : [];
+                    this.value = BigNumber.isBigNumber(this.value) ? Utils.strToByteArray(this.value.toFixed()) : [];
                     break;
                 case Dish.JSON:
                     this.value = this.value ? Utils.strToByteArray(JSON.stringify(this.value, null, 4)) : [];
@@ -265,7 +265,7 @@ class Dish {
             case Dish.ARRAY_BUFFER:
                 return this.value instanceof ArrayBuffer;
             case Dish.BIG_NUMBER:
-                return this.value instanceof BigNumber;
+                return BigNumber.isBigNumber(this.value);
             case Dish.JSON:
                 // All values can be serialised in some manner, so we return true in all cases
                 return true;

src/core/Ingredient.mjs

@@ -23,6 +23,7 @@ class Ingredient {
         this._value = null;
         this.disabled = false;
         this.hint = "";
+        this.rows = 0;
         this.toggleValues = [];
         this.target = null;
         this.defaultIndex = 0;
@@ -45,6 +46,7 @@ class Ingredient {
         this.defaultValue = ingredientConfig.value;
         this.disabled = !!ingredientConfig.disabled;
         this.hint = ingredientConfig.hint || false;
+        this.rows = ingredientConfig.rows || false;
         this.toggleValues = ingredientConfig.toggleValues;
         this.target = typeof ingredientConfig.target !== "undefined" ? ingredientConfig.target : null;
         this.defaultIndex = typeof ingredientConfig.defaultIndex !== "undefined" ? ingredientConfig.defaultIndex : 0;

src/core/Operation.mjs

@@ -23,6 +23,7 @@ class Operation {
         this._breakpoint      = false;
         this._disabled        = false;
         this._flowControl     = false;
+        this._manualBake      = false;
         this._ingList         = [];
 
         // Public fields
@@ -179,6 +180,7 @@ class Operation {
 
             if (ing.toggleValues) conf.toggleValues = ing.toggleValues;
             if (ing.hint) conf.hint = ing.hint;
+            if (ing.rows) conf.rows = ing.rows;
             if (ing.disabled) conf.disabled = ing.disabled;
             if (ing.target) conf.target = ing.target;
             if (ing.defaultIndex) conf.defaultIndex = ing.defaultIndex;
@@ -281,6 +283,7 @@ class Operation {
         return this._flowControl;
     }
 
+
     /**
      * Set whether this Operation is a flowcontrol op.
      *
@@ -290,6 +293,26 @@ class Operation {
         this._flowControl = !!value;
     }
 
+
+    /**
+     * Returns true if this Operation should not trigger AutoBake.
+     *
+     * @returns {boolean}
+     */
+    get manualBake() {
+        return this._manualBake;
+    }
+
+
+    /**
+     * Set whether this Operation should trigger AutoBake.
+     *
+     * @param {boolean} value
+     */
+    set manualBake(value) {
+        this._manualBake = !!value;
+    }
+
 }
 
 export default Operation;

src/core/config/Categories.json

@@ -155,6 +155,7 @@
         "name": "Networking",
         "ops": [
             "HTTP request",
+            "DNS over HTTPS",
             "Strip HTTP headers",
             "Dechunk HTTP response",
             "Parse User Agent",
@@ -215,6 +216,7 @@
             "Convert mass",
             "Convert speed",
             "Convert data units",
+            "Convert co-ordinate format",
             "Parse UNIX file permissions",
             "Swap endianness",
             "Parse colour code",
@@ -306,9 +308,7 @@
             "Adler-32 Checksum",
             "CRC-16 Checksum",
             "CRC-32 Checksum",
-            "TCP/IP Checksum",
-            "To Geohash",
-            "From Geohash"
+            "TCP/IP Checksum"
         ]
     },
     {

src/core/config/scripts/generateConfig.mjs

@@ -41,6 +41,7 @@ for (const opObj in Ops) {
         inputType:   op.inputType,
         outputType:  op.presentType,
         flowControl: op.flowControl,
+        manualBake:  op.manualBake,
         args:        op.args
     };
 

src/core/lib/ConvertCoordinates.mjs

@@ -0,0 +1,655 @@
+/**
+ * Co-ordinate conversion resources.
+ *
+ * @author j433866 [j433866@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import geohash from "ngeohash";
+import geodesy from "geodesy";
+import OperationError from "../errors/OperationError";
+
+/**
+ * Co-ordinate formats
+ */
+export const FORMATS = [
+    "Degrees Minutes Seconds",
+    "Degrees Decimal Minutes",
+    "Decimal Degrees",
+    "Geohash",
+    "Military Grid Reference System",
+    "Ordnance Survey National Grid",
+    "Universal Transverse Mercator"
+];
+
+/**
+ * Formats that should be passed to the conversion module as-is
+ */
+const NO_CHANGE = [
+    "Geohash",
+    "Military Grid Reference System",
+    "Ordnance Survey National Grid",
+    "Universal Transverse Mercator",
+];
+
+/**
+ * Convert a given latitude and longitude into a different format.
+ *
+ * @param {string} input - Input string to be converted
+ * @param {string} inFormat - Format of the input coordinates
+ * @param {string} inDelim - The delimiter splitting the lat/long of the input
+ * @param {string} outFormat - Format to convert to
+ * @param {string} outDelim - The delimiter to separate the output with
+ * @param {string} includeDir - Whether or not to include the compass direction in the output
+ * @param {number} precision - Precision of the result
+ * @returns {string} A formatted string of the converted co-ordinates
+ */
+export function convertCoordinates (input, inFormat, inDelim, outFormat, outDelim, includeDir, precision) {
+    let isPair = false,
+        split,
+        latlon,
+        convLat,
+        convLon,
+        conv,
+        hash,
+        utm,
+        mgrs,
+        osng,
+        splitLat,
+        splitLong,
+        lat,
+        lon;
+
+    // Can't have a precision less than 0!
+    if (precision < 0) {
+        precision = 0;
+    }
+
+    if (inDelim === "Auto") {
+        // Try to detect a delimiter in the input.
+        inDelim = findDelim(input);
+        if (inDelim === null) {
+            throw new OperationError("Unable to detect the input delimiter automatically.");
+        }
+    } else if (!inDelim.includes("Direction")) {
+        // Convert the delimiter argument value to the actual character
+        inDelim = realDelim(inDelim);
+    }
+
+    if (inFormat === "Auto") {
+        // Try to detect the format of the input data
+        inFormat = findFormat(input, inDelim);
+        if (inFormat === null) {
+            throw new OperationError("Unable to detect the input format automatically.");
+        }
+    }
+
+    // Convert the output delimiter argument to the real character
+    outDelim = realDelim(outDelim);
+
+    if (!NO_CHANGE.includes(inFormat)) {
+        if (inDelim.includes("Direction")) {
+            // Split on directions
+            split = input.split(/[NnEeSsWw]/g);
+            if (split[0] === "") {
+                // Remove first element if direction preceding
+                split = split.slice(1);
+            }
+        } else {
+            split = input.split(inDelim);
+        }
+        // Replace any co-ordinate symbols with spaces so we can split on them later
+        for (let i = 0; i < split.length; i++) {
+            split[i] = split[i].replace(/[°˝´'"]/g, " ");
+        }
+        if (split.length > 1) {
+            isPair = true;
+        }
+    } else {
+        // Remove any delimiters from the input
+        input = input.replace(inDelim, "");
+        isPair = true;
+    }
+
+    // Conversions from the input format into a geodesy latlon object
+    switch (inFormat) {
+        case "Geohash":
+            hash = geohash.decode(input.replace(/[^A-Za-z0-9]/g, ""));
+            latlon = new geodesy.LatLonEllipsoidal(hash.latitude, hash.longitude);
+            break;
+        case "Military Grid Reference System":
+            utm = geodesy.Mgrs.parse(input.replace(/[^A-Za-z0-9]/g, "")).toUtm();
+            latlon = utm.toLatLonE();
+            break;
+        case "Ordnance Survey National Grid":
+            osng = geodesy.OsGridRef.parse(input.replace(/[^A-Za-z0-9]/g, ""));
+            latlon = geodesy.OsGridRef.osGridToLatLon(osng);
+            break;
+        case "Universal Transverse Mercator":
+            // Geodesy needs a space between the first 2 digits and the next letter
+            if (/^[\d]{2}[A-Za-z]/.test(input)) {
+                input = input.slice(0, 2) + " " + input.slice(2);
+            }
+            utm = geodesy.Utm.parse(input);
+            latlon = utm.toLatLonE();
+            break;
+        case "Degrees Minutes Seconds":
+            if (isPair) {
+                // Split up the lat/long into degrees / minutes / seconds values
+                splitLat = splitInput(split[0]);
+                splitLong = splitInput(split[1]);
+
+                if (splitLat.length >= 3 && splitLong.length >= 3) {
+                    lat = convDMSToDD(splitLat[0], splitLat[1], splitLat[2], 10);
+                    lon = convDMSToDD(splitLong[0], splitLong[1], splitLong[2], 10);
+                    latlon = new geodesy.LatLonEllipsoidal(lat.degrees, lon.degrees);
+                } else {
+                    throw new OperationError("Invalid co-ordinate format for Degrees Minutes Seconds");
+                }
+            } else {
+                // Not a pair, so only try to convert one set of co-ordinates
+                splitLat = splitInput(split[0]);
+                if (splitLat.length >= 3) {
+                    lat = convDMSToDD(splitLat[0], splitLat[1], splitLat[2]);
+                    latlon = new geodesy.LatLonEllipsoidal(lat.degrees, lat.degrees);
+                } else {
+                    throw new OperationError("Invalid co-ordinate format for Degrees Minutes Seconds");
+                }
+            }
+            break;
+        case "Degrees Decimal Minutes":
+            if (isPair) {
+                splitLat = splitInput(split[0]);
+                splitLong = splitInput(split[1]);
+                if (splitLat.length !== 2 || splitLong.length !== 2) {
+                    throw new OperationError("Invalid co-ordinate format for Degrees Decimal Minutes.");
+                }
+                // Convert to decimal degrees, and then convert to a geodesy object
+                lat = convDDMToDD(splitLat[0], splitLat[1], 10);
+                lon = convDDMToDD(splitLong[0], splitLong[1], 10);
+                latlon = new geodesy.LatLonEllipsoidal(lat.degrees, lon.degrees);
+            } else {
+                // Not a pair, so only try to convert one set of co-ordinates
+                splitLat = splitInput(input);
+                if (splitLat.length !== 2) {
+                    throw new OperationError("Invalid co-ordinate format for Degrees Decimal Minutes.");
+                }
+                lat = convDDMToDD(splitLat[0], splitLat[1], 10);
+                latlon = new geodesy.LatLonEllipsoidal(lat.degrees, lat.degrees);
+            }
+            break;
+        case "Decimal Degrees":
+            if (isPair) {
+                splitLat =  splitInput(split[0]);
+                splitLong = splitInput(split[1]);
+                if (splitLat.length !== 1 || splitLong.length !== 1) {
+                    throw new OperationError("Invalid co-ordinate format for Decimal Degrees.");
+                }
+                latlon = new geodesy.LatLonEllipsoidal(splitLat[0], splitLong[0]);
+            } else {
+                // Not a pair, so only try to convert one set of co-ordinates
+                splitLat = splitInput(split[0]);
+                if (splitLat.length !== 1) {
+                    throw new OperationError("Invalid co-ordinate format for Decimal Degrees.");
+                }
+                latlon = new geodesy.LatLonEllipsoidal(splitLat[0], splitLat[0]);
+            }
+            break;
+        default:
+            throw new OperationError(`Unknown input format '${inFormat}'`);
+    }
+
+    // Everything is now a geodesy latlon object
+    // These store the latitude and longitude as decimal
+    if (inFormat.includes("Degrees")) {
+        // If the input string contains directions, we need to check if they're S or W.
+        // If either of the directions are, we should make the decimal value negative
+        const dirs = input.toUpperCase().match(/[NESW]/g);
+        if (dirs && dirs.length >= 1) {
+            // Make positive lat/lon values with S/W directions into negative values
+            if (dirs[0] === "S" || dirs[0] === "W" && latlon.lat > 0) {
+                latlon.lat = -latlon.lat;
+            }
+            if (dirs.length >= 2) {
+                if (dirs[1] === "S" || dirs[1] === "W" && latlon.lon > 0) {
+                    latlon.lon = -latlon.lon;
+                }
+            }
+        }
+    }
+
+    // Try to find the compass directions of the lat and long
+    const [latDir, longDir] = findDirs(latlon.lat + "," + latlon.lon, ",");
+
+    // Output conversions for each output format
+    switch (outFormat) {
+        case "Decimal Degrees":
+            // We could use the built in latlon.toString(),
+            // but this makes adjusting the output harder
+            lat = convDDToDD(latlon.lat, precision);
+            lon = convDDToDD(latlon.lon, precision);
+            convLat = lat.string;
+            convLon = lon.string;
+            break;
+        case "Degrees Decimal Minutes":
+            lat = convDDToDDM(latlon.lat, precision);
+            lon = convDDToDDM(latlon.lon, precision);
+            convLat = lat.string;
+            convLon = lon.string;
+            break;
+        case "Degrees Minutes Seconds":
+            lat = convDDToDMS(latlon.lat, precision);
+            lon = convDDToDMS(latlon.lon, precision);
+            convLat = lat.string;
+            convLon = lon.string;
+            break;
+        case "Geohash":
+            convLat = geohash.encode(latlon.lat, latlon.lon, precision);
+            break;
+        case "Military Grid Reference System":
+            utm = latlon.toUtm();
+            mgrs = utm.toMgrs();
+            // MGRS wants a precision that's an even number between 2 and 10
+            if (precision % 2 !== 0) {
+                precision = precision + 1;
+            }
+            if (precision > 10) {
+                precision = 10;
+            }
+            convLat = mgrs.toString(precision);
+            break;
+        case "Ordnance Survey National Grid":
+            osng = geodesy.OsGridRef.latLonToOsGrid(latlon);
+            if (osng.toString() === "") {
+                throw new OperationError("Could not convert co-ordinates to OS National Grid. Are the co-ordinates in range?");
+            }
+            // OSNG wants a precision that's an even number between 2 and 10
+            if (precision % 2 !== 0) {
+                precision = precision + 1;
+            }
+            if (precision > 10) {
+                precision = 10;
+            }
+            convLat = osng.toString(precision);
+            break;
+        case "Universal Transverse Mercator":
+            utm = latlon.toUtm();
+            convLat = utm.toString(precision);
+            break;
+    }
+
+    if (convLat === undefined) {
+        throw new OperationError("Error converting co-ordinates.");
+    }
+
+    if (outFormat.includes("Degrees")) {
+        // Format DD/DDM/DMS for output
+        // If we're outputting a compass direction, remove the negative sign
+        if (latDir === "S" && includeDir !== "None") {
+            convLat = convLat.replace("-", "");
+        }
+        if (longDir === "W" && includeDir !== "None") {
+            convLon = convLon.replace("-", "");
+        }
+
+        let outConv = "";
+        if (includeDir === "Before") {
+            outConv += latDir + " ";
+        }
+
+        outConv += convLat;
+        if (includeDir === "After") {
+            outConv += " " + latDir;
+        }
+        outConv += outDelim;
+        if (isPair) {
+            if (includeDir === "Before") {
+                outConv += longDir + " ";
+            }
+            outConv += convLon;
+            if (includeDir === "After") {
+                outConv += " " + longDir;
+            }
+            outConv += outDelim;
+        }
+        conv = outConv;
+    } else {
+        conv = convLat + outDelim;
+    }
+
+    return conv;
+}
+
+/**
+ * Split up the input using a space or degrees signs, and sanitise the result
+ *
+ * @param {string} input - The input data to be split
+ * @returns {number[]} An array of the different items in the string, stored as floats
+ */
+function splitInput (input){
+    const split = [];
+
+    input.split(/\s+/).forEach(item => {
+        // Remove any character that isn't a digit, decimal point or negative sign
+        item = item.replace(/[^0-9.-]/g, "");
+        if (item.length > 0){
+            // Turn the item into a float
+            split.push(parseFloat(item));
+        }
+    });
+    return split;
+}
+
+/**
+ * Convert Degrees Minutes Seconds to Decimal Degrees
+ *
+ * @param {number} degrees - The degrees of the input co-ordinates
+ * @param {number} minutes - The minutes of the input co-ordinates
+ * @param {number} seconds - The seconds of the input co-ordinates
+ * @param {number} precision - The precision the result should be rounded to
+ * @returns {{string: string, degrees: number}} An object containing the raw converted value (obj.degrees), and a formatted string version (obj.string)
+ */
+function convDMSToDD (degrees, minutes, seconds, precision){
+    const absDegrees = Math.abs(degrees);
+    let conv = absDegrees + (minutes / 60) + (seconds / 3600);
+    let outString = round(conv, precision) + "°";
+    if (isNegativeZero(degrees) || degrees < 0) {
+        conv = -conv;
+        outString = "-" + outString;
+    }
+    return {
+        "degrees": conv,
+        "string": outString
+    };
+}
+
+/**
+ * Convert Decimal Degrees Minutes to Decimal Degrees
+ *
+ * @param {number} degrees - The input degrees to be converted
+ * @param {number} minutes - The input minutes to be converted
+ * @param {number} precision - The precision which the result should be rounded to
+ * @returns {{string: string, degrees: number}} An object containing the raw converted value (obj.degrees), and a formatted string version (obj.string)
+ */
+function convDDMToDD (degrees, minutes, precision) {
+    const absDegrees = Math.abs(degrees);
+    let conv = absDegrees + minutes / 60;
+    let outString = round(conv, precision) + "°";
+    if (isNegativeZero(degrees) || degrees < 0) {
+        conv = -conv;
+        outString = "-" + outString;
+    }
+    return {
+        "degrees": conv,
+        "string": outString
+    };
+}
+
+/**
+ * Convert Decimal Degrees to Decimal Degrees
+ *
+ * Doesn't affect the input, just puts it into an object
+ * @param {number} degrees - The input degrees to be converted
+ * @param {number} precision - The precision which the result should be rounded to
+ * @returns {{string: string, degrees: number}} An object containing the raw converted value (obj.degrees), and a formatted string version (obj.string)
+ */
+function convDDToDD (degrees, precision) {
+    return {
+        "degrees": degrees,
+        "string": round(degrees, precision) + "°"
+    };
+}
+
+/**
+ * Convert Decimal Degrees to Degrees Minutes Seconds
+ *
+ * @param {number} decDegrees - The input data to be converted
+ * @param {number} precision - The precision which the result should be rounded to
+ * @returns {{string: string, degrees: number, minutes: number, seconds: number}} An object containing the raw converted value as separate numbers (.degrees, .minutes, .seconds), and a formatted string version (obj.string)
+ */
+function convDDToDMS (decDegrees, precision) {
+    const absDegrees = Math.abs(decDegrees);
+    let degrees = Math.floor(absDegrees);
+    const minutes = Math.floor(60 * (absDegrees - degrees)),
+        seconds = round(3600 * (absDegrees - degrees) - 60 * minutes, precision);
+    let outString = degrees + "° " + minutes + "' " + seconds + "\"";
+    if (isNegativeZero(decDegrees) || decDegrees < 0) {
+        degrees = -degrees;
+        outString = "-" + outString;
+    }
+    return {
+        "degrees": degrees,
+        "minutes": minutes,
+        "seconds": seconds,
+        "string": outString
+    };
+}
+
+/**
+ * Convert Decimal Degrees to Degrees Decimal Minutes
+ *
+ * @param {number} decDegrees - The input degrees to be converted
+ * @param {number} precision - The precision the input data should be rounded to
+ * @returns {{string: string, degrees: number, minutes: number}} An object containing the raw converted value as separate numbers (.degrees, .minutes), and a formatted string version (obj.string)
+ */
+function convDDToDDM (decDegrees, precision) {
+    const absDegrees = Math.abs(decDegrees);
+    let degrees = Math.floor(absDegrees);
+    const minutes = absDegrees - degrees,
+        decMinutes = round(minutes * 60, precision);
+    let outString = degrees + "° " + decMinutes + "'";
+    if (decDegrees < 0 || isNegativeZero(decDegrees)) {
+        degrees = -degrees;
+        outString = "-" + outString;
+    }
+
+    return {
+        "degrees": degrees,
+        "minutes": decMinutes,
+        "string": outString,
+    };
+}
+
+/**
+ * Finds and returns the compass directions in an input string
+ *
+ * @param {string} input - The input co-ordinates containing the direction
+ * @param {string} delim - The delimiter separating latitide and longitude
+ * @returns {string[]} String array containing the latitude and longitude directions
+ */
+export function findDirs(input, delim) {
+    const upperInput = input.toUpperCase();
+    const dirExp = new RegExp(/[NESW]/g);
+
+    const dirs = upperInput.match(dirExp);
+
+    if (dirs) {
+        // If there's actually compass directions
+        // in the input, use these to work out the direction
+        if (dirs.length <= 2 && dirs.length >= 1) {
+            return dirs.length === 2 ? [dirs[0], dirs[1]] : [dirs[0], ""];
+        }
+    }
+
+    // Nothing was returned, so guess the directions
+    let lat = upperInput,
+        long,
+        latDir = "",
+        longDir = "";
+    if (!delim.includes("Direction")) {
+        if (upperInput.includes(delim)) {
+            const split = upperInput.split(delim);
+            if (split.length >= 1) {
+                if (split[0] !== "") {
+                    lat = split[0];
+                }
+                if (split.length >= 2 && split[1] !== "") {
+                    long = split[1];
+                }
+            }
+        }
+    } else {
+        const split = upperInput.split(dirExp);
+        if (split.length > 1) {
+            lat = split[0] === "" ? split[1] : split[0];
+            if (split.length > 2 && split[2] !== "") {
+                long = split[2];
+            }
+        }
+    }
+
+    if (lat) {
+        lat = parseFloat(lat);
+        latDir = lat < 0 ? "S" : "N";
+    }
+
+    if (long) {
+        long = parseFloat(long);
+        longDir = long < 0 ? "W" : "E";
+    }
+
+    return [latDir, longDir];
+}
+
+/**
+ * Detects the co-ordinate format of the input data
+ *
+ * @param {string} input - The input data whose format we need to detect
+ * @param {string} delim - The delimiter separating the data in input
+ * @returns {string} The input format
+ */
+export function findFormat (input, delim) {
+    let testData;
+    const mgrsPattern = new RegExp(/^[0-9]{2}\s?[C-HJ-NP-X]{1}\s?[A-HJ-NP-Z][A-HJ-NP-V]\s?[0-9\s]+/),
+        osngPattern = new RegExp(/^[A-HJ-Z]{2}\s+[0-9\s]+$/),
+        geohashPattern = new RegExp(/^[0123456789BCDEFGHJKMNPQRSTUVWXYZ]+$/),
+        utmPattern = new RegExp(/^[0-9]{2}\s?[C-HJ-NP-X]\s[0-9.]+\s?[0-9.]+$/),
+        degPattern = new RegExp(/[°'"]/g);
+
+    input = input.trim();
+
+    if (delim !== null && delim.includes("Direction")) {
+        const split = input.split(/[NnEeSsWw]/);
+        if (split.length > 1) {
+            testData = split[0] === "" ? split[1] : split[0];
+        }
+    } else if (delim !== null && delim !== "") {
+        if (input.includes(delim)) {
+            const split = input.split(delim);
+            if (split.length > 1) {
+                testData = split[0] === "" ? split[1] : split[0];
+            }
+        } else {
+            testData = input;
+        }
+    }
+
+    // Test non-degrees formats
+    if (!degPattern.test(input)) {
+        const filteredInput = input.toUpperCase().replace(delim, "");
+
+        if (utmPattern.test(filteredInput)) {
+            return "Universal Transverse Mercator";
+        }
+        if (mgrsPattern.test(filteredInput)) {
+            return "Military Grid Reference System";
+        }
+        if (osngPattern.test(filteredInput)) {
+            return "Ordnance Survey National Grid";
+        }
+        if (geohashPattern.test(filteredInput)) {
+            return "Geohash";
+        }
+    }
+
+    // Test DMS/DDM/DD formats
+    if (testData !== undefined) {
+        const split = splitInput(testData);
+        switch (split.length){
+            case 3:
+                return "Degrees Minutes Seconds";
+            case 2:
+                return "Degrees Decimal Minutes";
+            case 1:
+                return "Decimal Degrees";
+        }
+    }
+    return null;
+}
+
+/**
+ * Automatically find the delimeter type from the given input
+ *
+ * @param {string} input
+ * @returns {string} Delimiter type
+ */
+export function findDelim (input) {
+    input = input.trim();
+    const delims = [",", ";", ":"];
+    const testDir = input.match(/[NnEeSsWw]/g);
+    if (testDir !== null && testDir.length > 0 && testDir.length < 3) {
+        // Possibly contains a direction
+        const splitInput = input.split(/[NnEeSsWw]/);
+        if (splitInput.length <= 3 && splitInput.length > 0) {
+            // If there's 3 splits (one should be empty), then assume we have directions
+            if (splitInput[0] === "") {
+                return "Direction Preceding";
+            } else if (splitInput[splitInput.length - 1] === "") {
+                return "Direction Following";
+            }
+        }
+    }
+
+    // Loop through the standard delimiters, and try to find them in the input
+    for (let i = 0; i < delims.length; i++) {
+        const delim = delims[i];
+        if (input.includes(delim)) {
+            const splitInput = input.split(delim);
+            if (splitInput.length <= 3 && splitInput.length > 0) {
+                // Don't want to try and convert more than 2 co-ordinates
+                return delim;
+            }
+        }
+    }
+    return null;
+}
+
+/**
+ * Gets the real string for a delimiter name.
+ *
+ * @param {string} delim The delimiter to be matched
+ * @returns {string}
+ */
+export function realDelim (delim) {
+    return {
+        "Auto":         "Auto",
+        "Space":        " ",
+        "\\n":          "\n",
+        "Comma":        ",",
+        "Semi-colon":   ";",
+        "Colon":        ":"
+    }[delim];
+}
+
+/**
+ * Returns true if a zero is negative
+ *
+ * @param {number} zero
+ * @returns {boolean}
+ */
+function isNegativeZero(zero) {
+    return zero === 0 && (1/zero < 0);
+}
+
+/**
+ * Rounds a number to a specified number of decimal places
+ *
+ * @param {number} input - The number to be rounded
+ * @param {precision} precision - The number of decimal places the number should be rounded to
+ * @returns {number}
+ */
+function round(input, precision) {
+    precision = Math.pow(10, precision);
+    return Math.round(input * precision) / precision;
+}

src/core/operations/ConvertCoordinateFormat.mjs

@@ -0,0 +1,95 @@
+/**
+ * @author j433866 [j433866@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import {FORMATS, convertCoordinates} from "../lib/ConvertCoordinates";
+
+/**
+ * Convert co-ordinate format operation
+ */
+class ConvertCoordinateFormat extends Operation {
+
+    /**
+     * ConvertCoordinateFormat constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Convert co-ordinate format";
+        this.module = "Hashing";
+        this.description = "Converts geographical coordinates between different formats.<br><br>Supported formats:<ul><li>Degrees Minutes Seconds (DMS)</li><li>Degrees Decimal Minutes (DDM)</li><li>Decimal Degrees (DD)</li><li>Geohash</li><li>Military Grid Reference System (MGRS)</li><li>Ordnance Survey National Grid (OSNG)</li><li>Universal Transverse Mercator (UTM)</li></ul><br>The operation can try to detect the input co-ordinate format and delimiter automatically, but this may not always work correctly.";
+        this.infoURL = "https://wikipedia.org/wiki/Geographic_coordinate_conversion";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [
+            {
+                "name": "Input Format",
+                "type": "option",
+                "value": ["Auto"].concat(FORMATS)
+            },
+            {
+                "name": "Input Delimiter",
+                "type": "option",
+                "value": [
+                    "Auto",
+                    "Direction Preceding",
+                    "Direction Following",
+                    "\\n",
+                    "Comma",
+                    "Semi-colon",
+                    "Colon"
+                ]
+            },
+            {
+                "name": "Output Format",
+                "type": "option",
+                "value": FORMATS
+            },
+            {
+                "name": "Output Delimiter",
+                "type": "option",
+                "value": [
+                    "Space",
+                    "\\n",
+                    "Comma",
+                    "Semi-colon",
+                    "Colon"
+                ]
+            },
+            {
+                "name": "Include Compass Directions",
+                "type": "option",
+                "value": [
+                    "None",
+                    "Before",
+                    "After"
+                ]
+            },
+            {
+                "name": "Precision",
+                "type": "number",
+                "value": 3
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        if (input.replace(/[\s+]/g, "") !== "") {
+            const [inFormat, inDelim, outFormat, outDelim, incDirection, precision] = args;
+            const result = convertCoordinates(input, inFormat, inDelim, outFormat, outDelim, incDirection, precision);
+            return result;
+        } else {
+            return input;
+        }
+    }
+}
+
+export default ConvertCoordinateFormat;

src/core/operations/DNSOverHTTPS.mjs

@@ -0,0 +1,125 @@
+/**
+ * @author h345983745
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+
+/**
+ * DNS over HTTPS operation
+ */
+class DNSOverHTTPS extends Operation {
+
+    /**
+     * DNSOverHTTPS constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "DNS over HTTPS";
+        this.module = "Default";
+        this.description = [
+            "Takes a single domain name and performs a DNS lookup using DNS over HTTPS.",
+            "<br><br>",
+            "By default, <a href='https://developers.cloudflare.com/1.1.1.1/dns-over-https/'>Cloudflare</a> and <a href='https://developers.google.com/speed/public-dns/docs/dns-over-https'>Google</a> DNS over HTTPS services are supported.",
+            "<br><br>",
+            "Can be used with any service that supports the GET parameters <code>name</code> and <code>type</code>."
+        ].join("\n");
+        this.infoURL = "https://wikipedia.org/wiki/DNS_over_HTTPS";
+        this.inputType = "string";
+        this.outputType = "JSON";
+        this.manualBake = true;
+        this.args = [
+            {
+                name: "Resolver",
+                type: "editableOption",
+                value: [
+                    {
+                        name: "Google",
+                        value: "https://dns.google.com/resolve"
+                    },
+                    {
+                        name: "Cloudflare",
+                        value: "https://cloudflare-dns.com/dns-query"
+                    }
+                ]
+            },
+            {
+                name: "Request Type",
+                type: "option",
+                value: [
+                    "A",
+                    "AAAA",
+                    "TXT",
+                    "MX",
+                    "DNSKEY",
+                    "NS"
+                ]
+            },
+            {
+                name: "Answer Data Only",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Validate DNSSEC",
+                type: "boolean",
+                value: true
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {JSON}
+     */
+    run(input, args) {
+        const [resolver, requestType, justAnswer, DNSSEC] = args;
+        let url = URL;
+        try {
+            url = new URL(resolver);
+        } catch (error) {
+            throw new OperationError(error.toString() +
+            "\n\nThis error could be caused by one of the following:\n" +
+            " - An invalid Resolver URL\n");
+        }
+        const params = {name: input, type: requestType, cd: DNSSEC};
+
+        url.search = new URLSearchParams(params);
+
+        return fetch(url, {headers: {"accept": "application/dns-json"}}).then(response => {
+            return response.json();
+        }).then(data => {
+            if (justAnswer) {
+                return extractData(data.Answer);
+            }
+            return data;
+        }).catch(e => {
+            throw new OperationError(`Error making request to ${url}\n${e.toString()}`);
+        });
+
+    }
+}
+
+/**
+ * Construct an array of just data from a DNS Answer section
+ *
+ * @private
+ * @param {JSON} data
+ * @returns {JSON}
+ */
+function extractData(data) {
+    if (typeof(data) == "undefined"){
+        return [];
+    } else {
+        const dataValues = [];
+        data.forEach(element => {
+            dataValues.push(element.data);
+        });
+        return dataValues;
+    }
+}
+
+export default DNSOverHTTPS;

src/core/operations/Divide.mjs

@@ -43,7 +43,7 @@ class Divide extends Operation {
      */
     run(input, args) {
         const val = div(createNumArray(input, args[0]));
-        return val instanceof BigNumber ? val : new BigNumber(NaN);
+        return BigNumber.isBigNumber(val) ? val : new BigNumber(NaN);
     }
 
 }

src/core/operations/FromGeohash.mjs

@@ -1,44 +0,0 @@
-/**
- * @author gchq77703 []
- * @copyright Crown Copyright 2018
- * @license Apache-2.0
- */
-
-import Operation from "../Operation";
-import geohash from "ngeohash";
-
-/**
- * From Geohash operation
- */
-class FromGeohash extends Operation {
-
-    /**
-     * FromGeohash constructor
-     */
-    constructor() {
-        super();
-
-        this.name = "From Geohash";
-        this.module = "Crypto";
-        this.description = "Converts Geohash strings into Lat/Long coordinates. For example, <code>ww8p1r4t8</code> becomes <code>37.8324,112.5584</code>.";
-        this.infoURL = "https://wikipedia.org/wiki/Geohash";
-        this.inputType = "string";
-        this.outputType = "string";
-        this.args = [];
-    }
-
-    /**
-     * @param {string} input
-     * @param {Object[]} args
-     * @returns {string}
-     */
-    run(input, args) {
-        return input.split("\n").map(line => {
-            const coords = geohash.decode(line);
-            return [coords.latitude, coords.longitude].join(",");
-        }).join("\n");
-    }
-
-}
-
-export default FromGeohash;

src/core/operations/Mean.mjs

@@ -43,7 +43,7 @@ class Mean extends Operation {
      */
     run(input, args) {
         const val = mean(createNumArray(input, args[0]));
-        return val instanceof BigNumber ? val : new BigNumber(NaN);
+        return BigNumber.isBigNumber(val) ? val : new BigNumber(NaN);
     }
 
 }

src/core/operations/Median.mjs

@@ -43,7 +43,7 @@ class Median extends Operation {
      */
     run(input, args) {
         const val = median(createNumArray(input, args[0]));
-        return val instanceof BigNumber ? val : new BigNumber(NaN);
+        return BigNumber.isBigNumber(val) ? val : new BigNumber(NaN);
     }
 
 }

src/core/operations/Multiply.mjs

@@ -44,7 +44,7 @@ class Multiply extends Operation {
      */
     run(input, args) {
         const val = multi(createNumArray(input, args[0]));
-        return val instanceof BigNumber ? val : new BigNumber(NaN);
+        return BigNumber.isBigNumber(val) ? val : new BigNumber(NaN);
     }
 
 }

src/core/operations/RegularExpression.mjs

@@ -240,7 +240,7 @@ function regexHighlight (input, regex, displayTotal) {
         if (groups.length) {
             title += "Groups:\n";
             for (let i = 0; i < groups.length; i++) {
-                title += `\t${i+1}: ${Utils.escapeHtml(groups[i])}\n`;
+                title += `\t${i+1}: ${Utils.escapeHtml(groups[i] || "")}\n`;
             }
         }
 

src/core/operations/StandardDeviation.mjs

@@ -44,7 +44,7 @@ class StandardDeviation extends Operation {
      */
     run(input, args) {
         const val = stdDev(createNumArray(input, args[0]));
-        return val instanceof BigNumber ? val : new BigNumber(NaN);
+        return BigNumber.isBigNumber(val) ? val : new BigNumber(NaN);
 
     }
 

src/core/operations/Subtract.mjs

@@ -44,7 +44,7 @@ class Subtract extends Operation {
      */
     run(input, args) {
         const val = sub(createNumArray(input, args[0]));
-        return val instanceof BigNumber ? val : new BigNumber(NaN);
+        return BigNumber.isBigNumber(val) ? val : new BigNumber(NaN);
     }
 
 }

src/core/operations/Sum.mjs

@@ -44,7 +44,7 @@ class Sum extends Operation {
      */
     run(input, args) {
         const val = sum(createNumArray(input, args[0]));
-        return val instanceof BigNumber ? val : new BigNumber(NaN);
+        return BigNumber.isBigNumber(val) ? val : new BigNumber(NaN);
     }
 
 }

src/core/operations/ToGeohash.mjs

@@ -1,53 +0,0 @@
-/**
- * @author gchq77703 []
- * @copyright Crown Copyright 2018
- * @license Apache-2.0
- */
-
-import Operation from "../Operation";
-import geohash from "ngeohash";
-
-/**
- * To Geohash operation
- */
-class ToGeohash extends Operation {
-
-    /**
-     * ToGeohash constructor
-     */
-    constructor() {
-        super();
-
-        this.name = "To Geohash";
-        this.module = "Crypto";
-        this.description = "Converts Lat/Long coordinates into a Geohash string.  For example, <code>37.8324,112.5584</code> becomes <code>ww8p1r4t8</code>.";
-        this.infoURL = "https://wikipedia.org/wiki/Geohash";
-        this.inputType = "string";
-        this.outputType = "string";
-        this.args = [
-            {
-                name: "Precision",
-                type: "number",
-                value: 9
-            }
-        ];
-    }
-
-    /**
-     * @param {string} input
-     * @param {Object[]} args
-     * @returns {string}
-     */
-    run(input, args) {
-        const [precision] = args;
-
-        return input.split("\n").map(line => {
-            line = line.replace(/ /g, "");
-            if (line === "") return "";
-            return geohash.encode(...line.split(",").map(num => parseFloat(num)), precision);
-        }).join("\n");
-    }
-
-}
-
-export default ToGeohash;

src/core/operations/ToTable.mjs

@@ -57,7 +57,7 @@ class ToTable extends Operation {
         const [cellDelims, rowDelims, firstRowHeader, format] = args;
 
         // Process the input into a nested array of elements.
-        const tableData = Utils.parseCSV(input, cellDelims.split(""), rowDelims.split(""));
+        const tableData = Utils.parseCSV(Utils.escapeHtml(input), cellDelims.split(""), rowDelims.split(""));
 
         if (!tableData.length) return "";
 

src/core/operations/YARARules.mjs

@@ -0,0 +1,122 @@
+/**
+ * @author Matt C [matt@artemisbot.uk]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import Yara from "libyara-wasm";
+
+/**
+ * YARA Rules operation
+ */
+class YARARules extends Operation {
+
+    /**
+     * YARARules constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "YARA Rules";
+        this.module = "Yara";
+        this.description = "YARA is a tool developed at VirusTotal, primarily aimed at helping malware researchers to identify and classify malware samples. It matches based on rules specified by the user containing textual or binary patterns and a boolean expression. For help on writing rules, see the <a href='https://yara.readthedocs.io/en/latest/writingrules.html'>YARA documentation.</a>";
+        this.infoURL = "https://wikipedia.org/wiki/YARA";
+        this.inputType = "ArrayBuffer";
+        this.outputType = "string";
+        this.args = [
+            {
+                name: "Rules",
+                type: "text",
+                value: "",
+                rows: 5
+            },
+            {
+                name: "Show strings",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Show string lengths",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Show metadata",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Show counts",
+                type: "boolean",
+                value: true
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        if (ENVIRONMENT_IS_WORKER())
+            self.sendStatusMessage("Instantiating YARA...");
+        const [rules, showStrings, showLengths, showMeta, showCounts] = args;
+        return new Promise((resolve, reject) => {
+            Yara().then(yara => {
+                if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Converting data for YARA.");
+                let matchString = "";
+
+                const inpArr = new Uint8Array(input); // Turns out embind knows that JS uint8array <==> C++ std::string
+
+                if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Running YARA matching.");
+
+                const resp = yara.run(inpArr, rules);
+
+                if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Processing data.");
+
+                if (resp.compileErrors.size() > 0) {
+                    for (let i = 0; i < resp.compileErrors.size(); i++) {
+                        const compileError = resp.compileErrors.get(i);
+                        if (!compileError.warning) {
+                            reject(new OperationError(`Error on line ${compileError.lineNumber}: ${compileError.message}`));
+                        } else {
+                            matchString += `Warning on line ${compileError.lineNumber}: ${compileError.message}`;
+                        }
+                    }
+                }
+                const matchedRules = resp.matchedRules;
+                for (let i = 0; i < matchedRules.size(); i++) {
+                    const rule = matchedRules.get(i);
+                    const matches = rule.resolvedMatches;
+                    let meta = "";
+                    if (showMeta && rule.metadata.size() > 0) {
+                        meta += " [";
+                        for (let j = 0; j < rule.metadata.size(); j++) {
+                            meta += `${rule.metadata.get(j).identifier}: ${rule.metadata.get(j).data}, `;
+                        }
+                        meta = meta.slice(0, -2) + "]";
+                    }
+                    const countString = showCounts ? `${matches.size()} time${matches.size() > 1 ? "s" : ""}` : "";
+                    if (matches.size() === 0 || !(showStrings || showLengths)) {
+                        matchString += `Input matches rule "${rule.ruleName}"${meta}${countString.length > 0 ? ` ${countString}`: ""}.\n`;
+                    } else {
+                        matchString += `Rule "${rule.ruleName}"${meta} matches (${countString}):\n`;
+                        for (let j = 0; j < matches.size(); j++) {
+                            const match = matches.get(j);
+                            if (showStrings || showLengths) {
+                                matchString += `Pos ${match.location}, ${showLengths ? `length ${match.matchLength}, ` : ""}identifier ${match.stringIdentifier}${showStrings ? `, data: "${match.data}"` : ""}\n`;
+                            }
+                        }
+                    }
+                }
+                resolve(matchString);
+            });
+        });
+    }
+
+}
+
+export default YARARules;

src/web/HTMLIngredient.mjs

@@ -4,6 +4,8 @@
  * @license Apache-2.0
  */
 
+import Utils from "../core/Utils";
+
 /**
  * Object to handle the creation of operation ingredients.
  */
@@ -25,6 +27,7 @@ class HTMLIngredient {
         this.value = config.value;
         this.disabled = config.disabled || false;
         this.hint = config.hint || false;
+        this.rows = config.rows || false;
         this.target = config.target;
         this.defaultIndex = config.defaultIndex || 0;
         this.toggleValues = config.toggleValues;
@@ -155,7 +158,7 @@ class HTMLIngredient {
                     } else if ((m = this.value[i].name.match(/\[\/([a-z0-9 -()^]+)\]/i))) {
                         html += "</optgroup>";
                     } else {
-                        html += `<option populate-value="${this.value[i].value}">${this.value[i].name}</option>`;
+                        html += `<option populate-value="${Utils.escapeHtml(this.value[i].value)}">${this.value[i].name}</option>`;
                     }
                 }
                 html += `</select>
@@ -229,6 +232,7 @@ class HTMLIngredient {
                         class="form-control arg"
                         id="${this.id}"
                         arg-name="${this.name}"
+                        rows="${this.rows ? this.rows : 3}"
                         ${this.disabled ? "disabled" : ""}>${this.value}</textarea>
                     ${this.hint ? "<span class='bmd-help'>" + this.hint + "</span>" : ""}
                 </div>`;

src/web/Manager.mjs

@@ -137,6 +137,9 @@ class Manager {
         this.addDynamicListener("#rec-list li.operation > div", "dblclick", this.recipe.operationChildDblclick, this.recipe);
         this.addDynamicListener("#rec-list .dropdown-menu.toggle-dropdown a", "click", this.recipe.dropdownToggleClick, this.recipe);
         this.addDynamicListener("#rec-list", "operationremove", this.recipe.opRemove.bind(this.recipe));
+        this.addDynamicListener("textarea.arg", "dragover", this.recipe.textArgDragover, this.recipe);
+        this.addDynamicListener("textarea.arg", "dragleave", this.recipe.textArgDragLeave, this.recipe);
+        this.addDynamicListener("textarea.arg", "drop", this.recipe.textArgDrop, this.recipe);
 
         // Input
         this.addMultiEventListener("#input-text", "keyup", this.input.inputChange, this.input);

src/web/OutputWaiter.mjs

@@ -478,7 +478,7 @@ class OutputWaiter {
      */
     showMagicButton(opSequence, result, recipeConfig) {
         const magicButton = document.getElementById("magic");
-        magicButton.setAttribute("data-original-title", `<i>${opSequence}</i> will produce <span class="data-text">"${Utils.truncate(result, 30)}"</span>`);
+        magicButton.setAttribute("data-original-title", `<i>${opSequence}</i> will produce <span class="data-text">"${Utils.escapeHtml(Utils.truncate(result), 30)}"</span>`);
         magicButton.setAttribute("data-recipe", JSON.stringify(recipeConfig), null, "");
         magicButton.classList.remove("hidden");
     }

src/web/RecipeWaiter.mjs

@@ -376,6 +376,7 @@ class RecipeWaiter {
         }
     }
 
+
     /**
      * Adds the specified operation to the recipe.
      *
@@ -454,6 +455,75 @@ class RecipeWaiter {
     }
 
 
+    /**
+     * Handler for text argument dragover events.
+     * Gives the user a visual cue to show that items can be dropped here.
+     *
+     * @param {event} e
+     */
+    textArgDragover (e) {
+        // This will be set if we're dragging an operation
+        if (e.dataTransfer.effectAllowed === "move")
+            return false;
+
+        e.stopPropagation();
+        e.preventDefault();
+        e.target.closest("textarea.arg").classList.add("dropping-file");
+    }
+
+
+    /**
+     * Handler for text argument dragleave events.
+     * Removes the visual cue.
+     *
+     * @param {event} e
+     */
+    textArgDragLeave (e) {
+        e.stopPropagation();
+        e.preventDefault();
+        e.target.classList.remove("dropping-file");
+    }
+
+
+    /**
+     * Handler for text argument drop events.
+     * Loads the dragged data into the argument textarea.
+     *
+     * @param {event} e
+     */
+    textArgDrop(e) {
+        // This will be set if we're dragging an operation
+        if (e.dataTransfer.effectAllowed === "move")
+            return false;
+
+        e.stopPropagation();
+        e.preventDefault();
+        const targ = e.target;
+        const file = e.dataTransfer.files[0];
+        const text = e.dataTransfer.getData("Text");
+
+        targ.classList.remove("dropping-file");
+
+        if (text) {
+            targ.value = text;
+            return;
+        }
+
+        if (file) {
+            const reader = new FileReader();
+            const self = this;
+            reader.onload = function (e) {
+                targ.value = e.target.result;
+                // Trigger floating label move
+                const changeEvent = new Event("change");
+                targ.dispatchEvent(changeEvent);
+                window.dispatchEvent(self.manager.statechange);
+            };
+            reader.readAsText(file);
+        }
+    }
+
+
     /**
      * Sets register values.
      *
@@ -479,6 +549,7 @@ class RecipeWaiter {
         op.insertAdjacentHTML("beforeend", registerListEl);
     }
 
+
     /**
      * Adjusts the number of ingredient columns as the width of the recipe changes.
      */

src/web/stylesheets/layout/_io.css

@@ -123,6 +123,7 @@
 
 .dropping-file {
     border: 5px dashed var(--drop-file-border-colour) !important;
+    margin: -5px;
 }
 
 #stale-indicator {

tests/operations/index.mjs

@@ -45,7 +45,6 @@ import "./tests/DateTime";
 import "./tests/ExtractEmailAddresses";
 import "./tests/Fork";
 import "./tests/FromDecimal";
-import "./tests/FromGeohash";
 import "./tests/Hash";
 import "./tests/HaversineDistance";
 import "./tests/Hexdump";
@@ -77,12 +76,13 @@ import "./tests/SetUnion";
 import "./tests/StrUtils";
 import "./tests/SymmetricDifference";
 import "./tests/TextEncodingBruteForce";
-import "./tests/ToGeohash";
 import "./tests/TranslateDateTimeFormat";
 import "./tests/Magic";
 import "./tests/ParseTLV";
 import "./tests/Media";
 import "./tests/ToFromInsensitiveRegex";
+import "./tests/YARA.mjs";
+import "./tests/ConvertCoordinateFormat";
 
 // Cannot test operations that use the File type yet
 //import "./tests/SplitColourChannels";

tests/operations/tests/ConvertCoordinateFormat.mjs

@@ -0,0 +1,211 @@
+/**
+ * Convert co-ordinate format tests
+ *
+ * @author j433866
+ *
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+/**
+ * TEST CO-ORDINATES
+ * DD: 51.504°,-0.126°,
+ * DDM: 51° 30.24',-0° 7.56',
+ * DMS: 51° 30' 14.4",-0° 7' 33.6",
+ * Geohash: gcpvj0h0x,
+ * MGRS: 30U XC 99455 09790,
+ * OSNG: TQ 30163 80005,
+ * UTM: 30N 699456 5709791,
+ */
+
+import TestRegister from "../TestRegister";
+
+TestRegister.addTests([
+    {
+        name: "Co-ordinates: From Decimal Degrees to Degrees Minutes Seconds",
+        input: "51.504°,-0.126°,",
+        expectedOutput: "51° 30' 14.4\",-0° 7' 33.6\",",
+        recipeConfig: [
+            {
+                op: "Convert co-ordinate format",
+                args: ["Decimal Degrees", "Comma", "Degrees Minutes Seconds", "Comma", "None", 1]
+            },
+        ],
+    },
+    {
+        name: "Co-ordinates: From Degrees Minutes Seconds to Decimal Degrees",
+        input: "51° 30' 14.4\",-0° 7' 33.6\",",
+        expectedOutput: "51.504°,-0.126°,",
+        recipeConfig: [
+            {
+                op: "Convert co-ordinate format",
+                args: ["Degrees Minutes Seconds", "Comma", "Decimal Degrees", "Comma", "None", 3]
+            },
+        ],
+    },
+    {
+        name: "Co-ordinates: From Decimal Degrees to Degrees Decimal Minutes",
+        input: "51.504°,-0.126°,",
+        expectedOutput: "51° 30.24',-0° 7.56',",
+        recipeConfig: [
+            {
+                op: "Convert co-ordinate format",
+                args: ["Decimal Degrees", "Comma", "Degrees Decimal Minutes", "Comma", "None", 2]
+            }
+        ]
+    },
+    {
+        name: "Co-ordinates: From Degrees Decimal Minutes to Decimal Degrees",
+        input: "51° 30.24',-0° 7.56',",
+        expectedOutput: "51.504°,-0.126°,",
+        recipeConfig: [
+            {
+                op: "Convert co-ordinate format",
+                args: ["Degrees Decimal Minutes", "Comma", "Decimal Degrees", "Comma", "None", 3]
+            }
+        ]
+    },
+    {
+        name: "Co-ordinates: From Decimal Degrees to Decimal Degrees",
+        input: "51.504°,-0.126°,",
+        expectedOutput: "51.504°,-0.126°,",
+        recipeConfig: [
+            {
+                op: "Convert co-ordinate format",
+                args: ["Decimal Degrees", "Comma", "Decimal Degrees", "Comma", "None", 3]
+            }
+        ]
+    },
+    {
+        name: "Co-ordinates: From Decimal Degrees to Geohash",
+        input: "51.504°,-0.126°,",
+        expectedOutput: "gcpvj0h0x,",
+        recipeConfig: [
+            {
+                op: "Convert co-ordinate format",
+                args: ["Decimal Degrees", "Comma", "Geohash", "Comma", "None", 9]
+            },
+        ],
+    },
+    {
+        name: "Co-ordinates: From Geohash to Decimal Degrees",
+        input: "gcpvj0h0x,",
+        expectedOutput: "51.504°,-0.126°,",
+        recipeConfig: [
+            {
+                op: "Convert co-ordinate format",
+                args: ["Geohash", "Comma", "Decimal Degrees", "Comma", "None", 3]
+            },
+        ],
+    },
+    {
+        name: "Co-ordinates: From Decimal Degrees to MGRS",
+        input: "51.504°,-0.126°,",
+        expectedOutput: "30U XC 99455 09790,",
+        recipeConfig: [
+            {
+                op: "Convert co-ordinate format",
+                args: ["Decimal Degrees", "Comma", "Military Grid Reference System", "Comma", "None", 10]
+            },
+        ],
+    },
+    {
+        name: "Co-ordinates: From MGRS to Decimal Degrees",
+        input: "30U XC 99455 09790,",
+        expectedOutput: "51.504°,-0.126°,",
+        recipeConfig: [
+            {
+                op: "Convert co-ordinate format",
+                args: ["Military Grid Reference System", "Comma", "Decimal Degrees", "Comma", "None", 3]
+            }
+        ]
+    },
+    {
+        name: "Co-ordinates: From Decimal Degrees to OSNG",
+        input: "51.504°,-0.126°,",
+        expectedOutput: "TQ 30163 80005,",
+        recipeConfig: [
+            {
+                op: "Convert co-ordinate format",
+                args: ["Decimal Degrees", "Comma", "Ordnance Survey National Grid", "Comma", "None", 10]
+            },
+        ],
+    },
+    {
+        name: "Co-ordinates: From OSNG to Decimal Degrees",
+        input: "TQ 30163 80005,",
+        expectedOutput: "51.504°,-0.126°,",
+        recipeConfig: [
+            {
+                op: "Convert co-ordinate format",
+                args: ["Ordnance Survey National Grid", "Comma", "Decimal Degrees", "Comma", "None", 3]
+            },
+        ],
+    },
+    {
+        name: "Co-ordinates: From Decimal Degrees to UTM",
+        input: "51.504°,-0.126°,",
+        expectedOutput: "30 N 699456 5709791,",
+        recipeConfig: [
+            {
+                op: "Convert co-ordinate format",
+                args: ["Decimal Degrees", "Comma", "Universal Transverse Mercator", "Comma", "None", 0]
+            },
+        ],
+    },
+    {
+        name: "Co-ordinates: From UTM to Decimal Degrees",
+        input: "30 N 699456 5709791,",
+        expectedOutput: "51.504°,-0.126°,",
+        recipeConfig: [
+            {
+                op: "Convert co-ordinate format",
+                args: ["Universal Transverse Mercator", "Comma", "Decimal Degrees", "Comma", "None", 3]
+            },
+        ],
+    },
+    {
+        name: "Co-ordinates: Directions in input, not output",
+        input: "N51.504°,W0.126°,",
+        expectedOutput: "51.504°,-0.126°,",
+        recipeConfig: [
+            {
+                op: "Convert co-ordinate format",
+                args: ["Decimal Degrees", "Comma", "Decimal Degrees", "Comma", "None", 3]
+            },
+        ],
+    },
+    {
+        name: "Co-ordinates: Directions in input and output",
+        input: "N51.504°,W0.126°,",
+        expectedOutput: "N 51.504°,W 0.126°,",
+        recipeConfig: [
+            {
+                op: "Convert co-ordinate format",
+                args: ["Decimal Degrees", "Comma", "Decimal Degrees", "Comma", "Before", 3]
+            },
+        ],
+    },
+    {
+        name: "Co-ordinates: Directions not in input, in output",
+        input: "51.504°,-0.126°,",
+        expectedOutput: "N 51.504°,W 0.126°,",
+        recipeConfig: [
+            {
+                op: "Convert co-ordinate format",
+                args: ["Decimal Degrees", "Comma", "Decimal Degrees", "Comma", "Before", 3]
+            },
+        ],
+    },
+    {
+        name: "Co-ordinates: Directions not in input, in converted output",
+        input: "51.504°,-0.126°,",
+        expectedOutput: "N 51° 30' 14.4\",W 0° 7' 33.6\",",
+        recipeConfig: [
+            {
+                op: "Convert co-ordinate format",
+                args: ["Decimal Degrees", "Comma", "Degrees Minutes Seconds", "Comma", "Before", 3]
+            },
+        ],
+    }
+]);

tests/operations/tests/FromGeohash.mjs

@@ -1,55 +0,0 @@
-/**
- * To Geohash tests
- *
- * @author gchq77703
- * @copyright Crown Copyright 2018
- * @license Apache-2.0
- */
-import TestRegister from "../TestRegister";
-
-TestRegister.addTests([
-    {
-        name: "From Geohash",
-        input: "ww8p1r4t8",
-        expectedOutput: "37.83238649368286,112.55838632583618",
-        recipeConfig: [
-            {
-                op: "From Geohash",
-                args: [],
-            },
-        ],
-    },
-    {
-        name: "From Geohash",
-        input: "ww8p1r",
-        expectedOutput: "37.83416748046875,112.5604248046875",
-        recipeConfig: [
-            {
-                op: "From Geohash",
-                args: [],
-            },
-        ],
-    },
-    {
-        name: "From Geohash",
-        input: "ww8",
-        expectedOutput: "37.265625,113.203125",
-        recipeConfig: [
-            {
-                op: "From Geohash",
-                args: [],
-            },
-        ],
-    },
-    {
-        name: "From Geohash",
-        input: "w",
-        expectedOutput: "22.5,112.5",
-        recipeConfig: [
-            {
-                op: "From Geohash",
-                args: [],
-            },
-        ],
-    },
-]);

tests/operations/tests/ToGeohash.mjs

@@ -1,55 +0,0 @@
-/**
- * To Geohash tests
- *
- * @author gchq77703
- * @copyright Crown Copyright 2018
- * @license Apache-2.0
- */
-import TestRegister from "../TestRegister";
-
-TestRegister.addTests([
-    {
-        name: "To Geohash",
-        input: "37.8324,112.5584",
-        expectedOutput: "ww8p1r4t8",
-        recipeConfig: [
-            {
-                op: "To Geohash",
-                args: [9],
-            },
-        ],
-    },
-    {
-        name: "To Geohash",
-        input: "37.9324,-112.2584",
-        expectedOutput: "9w8pv3ruj",
-        recipeConfig: [
-            {
-                op: "To Geohash",
-                args: [9],
-            },
-        ],
-    },
-    {
-        name: "To Geohash",
-        input: "37.8324,112.5584",
-        expectedOutput: "ww8",
-        recipeConfig: [
-            {
-                op: "To Geohash",
-                args: [3],
-            },
-        ],
-    },
-    {
-        name: "To Geohash",
-        input: "37.9324,-112.2584",
-        expectedOutput: "9w8pv3rujxy5b99",
-        recipeConfig: [
-            {
-                op: "To Geohash",
-                args: [15],
-            },
-        ],
-    },
-]);

tests/operations/tests/YARA.mjs

@@ -0,0 +1,24 @@
+/**
+ * YARA Rules tests.
+ *
+ * @author Matt C [matt@artemisbot.uk]
+ *
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+import TestRegister from "../TestRegister";
+
+TestRegister.addTests([
+    {
+        name: "YARA Match: simple foobar",
+        input: "foobar foobar bar foo foobar",
+        expectedOutput: "Rule \"foo\" matches (4 times):\nPos 0, length 3, identifier $re1, data: \"foo\"\nPos 7, length 3, identifier $re1, data: \"foo\"\nPos 18, length 3, identifier $re1, data: \"foo\"\nPos 22, length 3, identifier $re1, data: \"foo\"\nRule \"bar\" matches (4 times):\nPos 3, length 3, identifier $re1, data: \"bar\"\nPos 10, length 3, identifier $re1, data: \"bar\"\nPos 14, length 3, identifier $re1, data: \"bar\"\nPos 25, length 3, identifier $re1, data: \"bar\"\n",
+        recipeConfig: [
+            {
+                "op": "YARA Rules",
+                "args": ["rule foo {strings: $re1 = /foo/ condition: $re1} rule bar {strings: $re1 = /bar/ condition: $re1}", true, true, true, true],
+            }
+        ],
+    },
+]);
+