feat: Active Learning Updates (#778)

Adds new active learning form
Moves active learning settings from project settings to here
Refactored and created activeLearningService

cocoSSDModel/classes.json

@@ -0,0 +1 @@
+[{"name":"/m/01g317","id":1,"displayName":"person"},{"name":"/m/0199g","id":2,"displayName":"bicycle"},{"name":"/m/0k4j","id":3,"displayName":"car"},{"name":"/m/04_sv","id":4,"displayName":"motorcycle"},{"name":"/m/05czz6l","id":5,"displayName":"airplane"},{"name":"/m/01bjv","id":6,"displayName":"bus"},{"name":"/m/07jdr","id":7,"displayName":"train"},{"name":"/m/07r04","id":8,"displayName":"truck"},{"name":"/m/019jd","id":9,"displayName":"boat"},{"name":"/m/015qff","id":10,"displayName":"traffic light"},{"name":"/m/01pns0","id":11,"displayName":"fire hydrant"},{"name":"/m/02pv19","id":13,"displayName":"stop sign"},{"name":"/m/015qbp","id":14,"displayName":"parking meter"},{"name":"/m/0cvnqh","id":15,"displayName":"bench"},{"name":"/m/015p6","id":16,"displayName":"bird"},{"name":"/m/01yrx","id":17,"displayName":"cat"},{"name":"/m/0bt9lr","id":18,"displayName":"dog"},{"name":"/m/03k3r","id":19,"displayName":"horse"},{"name":"/m/07bgp","id":20,"displayName":"sheep"},{"name":"/m/01xq0k1","id":21,"displayName":"cow"},{"name":"/m/0bwd_0j","id":22,"displayName":"elephant"},{"name":"/m/01dws","id":23,"displayName":"bear"},{"name":"/m/0898b","id":24,"displayName":"zebra"},{"name":"/m/03bk1","id":25,"displayName":"giraffe"},{"name":"/m/01940j","id":27,"displayName":"backpack"},{"name":"/m/0hnnb","id":28,"displayName":"umbrella"},{"name":"/m/080hkjn","id":31,"displayName":"handbag"},{"name":"/m/01rkbr","id":32,"displayName":"tie"},{"name":"/m/01s55n","id":33,"displayName":"suitcase"},{"name":"/m/02wmf","id":34,"displayName":"frisbee"},{"name":"/m/071p9","id":35,"displayName":"skis"},{"name":"/m/06__v","id":36,"displayName":"snowboard"},{"name":"/m/018xm","id":37,"displayName":"sports ball"},{"name":"/m/02zt3","id":38,"displayName":"kite"},{"name":"/m/03g8mr","id":39,"displayName":"baseball bat"},{"name":"/m/03grzl","id":40,"displayName":"baseball glove"},{"name":"/m/06_fw","id":41,"displayName":"skateboard"},{"name":"/m/019w40","id":42,"displayName":"surfboard"},{"name":"/m/0dv9c","id":43,"displayName":"tennis racket"},{"name":"/m/04dr76w","id":44,"displayName":"bottle"},{"name":"/m/09tvcd","id":46,"displayName":"wine glass"},{"name":"/m/08gqpm","id":47,"displayName":"cup"},{"name":"/m/0dt3t","id":48,"displayName":"fork"},{"name":"/m/04ctx","id":49,"displayName":"knife"},{"name":"/m/0cmx8","id":50,"displayName":"spoon"},{"name":"/m/04kkgm","id":51,"displayName":"bowl"},{"name":"/m/09qck","id":52,"displayName":"banana"},{"name":"/m/014j1m","id":53,"displayName":"apple"},{"name":"/m/0l515","id":54,"displayName":"sandwich"},{"name":"/m/0cyhj_","id":55,"displayName":"orange"},{"name":"/m/0hkxq","id":56,"displayName":"broccoli"},{"name":"/m/0fj52s","id":57,"displayName":"carrot"},{"name":"/m/01b9xk","id":58,"displayName":"hot dog"},{"name":"/m/0663v","id":59,"displayName":"pizza"},{"name":"/m/0jy4k","id":60,"displayName":"donut"},{"name":"/m/0fszt","id":61,"displayName":"cake"},{"name":"/m/01mzpv","id":62,"displayName":"chair"},{"name":"/m/02crq1","id":63,"displayName":"couch"},{"name":"/m/03fp41","id":64,"displayName":"potted plant"},{"name":"/m/03ssj5","id":65,"displayName":"bed"},{"name":"/m/04bcr3","id":67,"displayName":"dining table"},{"name":"/m/09g1w","id":70,"displayName":"toilet"},{"name":"/m/07c52","id":72,"displayName":"tv"},{"name":"/m/01c648","id":73,"displayName":"laptop"},{"name":"/m/020lf","id":74,"displayName":"mouse"},{"name":"/m/0qjjc","id":75,"displayName":"remote"},{"name":"/m/01m2v","id":76,"displayName":"keyboard"},{"name":"/m/050k8","id":77,"displayName":"cell phone"},{"name":"/m/0fx9l","id":78,"displayName":"microwave"},{"name":"/m/029bxz","id":79,"displayName":"oven"},{"name":"/m/01k6s3","id":80,"displayName":"toaster"},{"name":"/m/0130jx","id":81,"displayName":"sink"},{"name":"/m/040b_t","id":82,"displayName":"refrigerator"},{"name":"/m/0bt_c3","id":84,"displayName":"book"},{"name":"/m/01x3z","id":85,"displayName":"clock"},{"name":"/m/02s195","id":86,"displayName":"vase"},{"name":"/m/01lsmm","id":87,"displayName":"scissors"},{"name":"/m/0kmg4","id":88,"displayName":"teddy bear"},{"name":"/m/03wvsk","id":89,"displayName":"hair drier"},{"name":"/m/012xff","id":90,"displayName":"toothbrush"}]

electron-builder.yml

@@ -20,3 +20,5 @@ linux:
     - snap
 publish: null
 electronVersion: 3.0.13
+extraFiles:
+  - "cocoSSDModel"

package-lock.json

@@ -1017,6 +1017,55 @@
                 "loader-utils": "^1.1.0"
             }
         },
+        "@tensorflow/tfjs": {
+            "version": "1.0.3",
+            "resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-1.0.3.tgz",
+            "integrity": "sha512-tF6GcjO2KBYlPPiS7o4X+D3oASXJcWAYaZA13GCYp5cXAui0ncHxpC85kmNQlp2HEVmcE82BJz/1uUtkNxxQpw==",
+            "requires": {
+                "@tensorflow/tfjs-converter": "1.0.3",
+                "@tensorflow/tfjs-core": "1.0.3",
+                "@tensorflow/tfjs-data": "1.0.3",
+                "@tensorflow/tfjs-layers": "1.0.3"
+            }
+        },
+        "@tensorflow/tfjs-converter": {
+            "version": "1.0.3",
+            "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-1.0.3.tgz",
+            "integrity": "sha512-vrGvVrPekhTOwMGsomcpcjw0ZUep6xhI8DQQoFXHjBcprt9bFO2hHMdAmYpqafcJ7KVMylbK4h2LJrsBI2zDgQ=="
+        },
+        "@tensorflow/tfjs-core": {
+            "version": "1.0.3",
+            "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-1.0.3.tgz",
+            "integrity": "sha512-2UbjMQkmrykIIZuoRfmDPrtWm+6fdQRlYLCUJdiOIooeu/q4nye587HM1qKcdZosGPZTW6VvX+4VIVieYn5i0A==",
+            "requires": {
+                "@types/seedrandom": "2.4.27",
+                "@types/webgl-ext": "0.0.30",
+                "@types/webgl2": "0.0.4",
+                "seedrandom": "2.4.3"
+            }
+        },
+        "@tensorflow/tfjs-data": {
+            "version": "1.0.3",
+            "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-data/-/tfjs-data-1.0.3.tgz",
+            "integrity": "sha512-WFjYU2pWNZ0TZaJ7rN18GD/wOTVe6rBGxvSwZxIhEVIbwKXaKXFa9V4aGp4QBG9AXHIA89SjmGSGPxfsC015hQ==",
+            "requires": {
+                "@types/node-fetch": "^2.1.2",
+                "node-fetch": "~2.1.2",
+                "seedrandom": "~2.4.3"
+            },
+            "dependencies": {
+                "node-fetch": {
+                    "version": "2.1.2",
+                    "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.1.2.tgz",
+                    "integrity": "sha1-q4hOjn5X44qUR1POxwb3iNF2i7U="
+                }
+            }
+        },
+        "@tensorflow/tfjs-layers": {
+            "version": "1.0.3",
+            "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-layers/-/tfjs-layers-1.0.3.tgz",
+            "integrity": "sha512-7VdvQb0ft7TrWAbBy7HI+p420KX9rblYYACZ7/BzvzsikfEOdEL90WxrZDjZ167rYN/KvqC/haGcmhW/dYU3MA=="
+        },
         "@types/axios": {
             "version": "0.14.0",
             "resolved": "https://registry.npmjs.org/@types/axios/-/axios-0.14.0.tgz",
@@ -1087,8 +1136,15 @@
         "@types/node": {
             "version": "10.12.7",
             "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.7.tgz",
-            "integrity": "sha512-Zh5Z4kACfbeE8aAOYh9mqotRxaZMro8MbBQtR8vEXOMiZo2rGEh2LayJijKdlu48YnS6y2EFU/oo2NCe5P6jGw==",
+            "integrity": "sha512-Zh5Z4kACfbeE8aAOYh9mqotRxaZMro8MbBQtR8vEXOMiZo2rGEh2LayJijKdlu48YnS6y2EFU/oo2NCe5P6jGw=="
-            "dev": true
+        },
+        "@types/node-fetch": {
+            "version": "2.1.7",
+            "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.1.7.tgz",
+            "integrity": "sha512-TZozHCDVrs0Aj1B9ZR5F4Q9MknDNcVd+hO5lxXOCzz07ELBey6s1gMUSZHUYHlPfRFKJFXiTnNuD7ePiI6S4/g==",
+            "requires": {
+                "@types/node": "*"
+            }
         },
         "@types/prop-types": {
             "version": "15.5.8",
@@ -1230,6 +1286,11 @@
                 "redux": "^4.0.0"
             }
         },
+        "@types/seedrandom": {
+            "version": "2.4.27",
+            "resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-2.4.27.tgz",
+            "integrity": "sha1-nbVjk33YaRX2kJK8QyWdL0hXjkE="
+        },
         "@types/snapsvg": {
             "version": "0.4.35",
             "resolved": "https://registry.npmjs.org/@types/snapsvg/-/snapsvg-0.4.35.tgz",
@@ -1243,6 +1304,16 @@
             "resolved": "https://registry.npmjs.org/@types/tapable/-/tapable-1.0.2.tgz",
             "integrity": "sha512-42zEJkBpNfMEAvWR5WlwtTH22oDzcMjFsL9gDGExwF8X8WvAiw7Vwop7hPw03QT8TKfec83LwbHj6SvpqM4ELQ=="
         },
+        "@types/webgl-ext": {
+            "version": "0.0.30",
+            "resolved": "https://registry.npmjs.org/@types/webgl-ext/-/webgl-ext-0.0.30.tgz",
+            "integrity": "sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg=="
+        },
+        "@types/webgl2": {
+            "version": "0.0.4",
+            "resolved": "https://registry.npmjs.org/@types/webgl2/-/webgl2-0.0.4.tgz",
+            "integrity": "sha512-PACt1xdErJbMUOUweSrbVM7gSIYm1vTncW2hF6Os/EeWi6TXYAYMPp+8v6rzHmypE5gHrxaxZNXgMkJVIdZpHw=="
+        },
         "@webassemblyjs/ast": {
             "version": "1.7.6",
             "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.7.6.tgz",
@@ -9380,6 +9451,17 @@
             "requires": {
                 "node-fetch": "^1.0.1",
                 "whatwg-fetch": ">=0.10.0"
+            },
+            "dependencies": {
+                "node-fetch": {
+                    "version": "1.7.3",
+                    "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz",
+                    "integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==",
+                    "requires": {
+                        "encoding": "^0.1.11",
+                        "is-stream": "^1.0.1"
+                    }
+                }
             }
         },
         "isstream": {
@@ -10150,6 +10232,11 @@
                 "topo": "2.x.x"
             }
         },
+        "jpeg-js": {
+            "version": "0.3.4",
+            "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.3.4.tgz",
+            "integrity": "sha512-6IzjQxvnlT8UlklNmDXIJMWxijULjqGrzgqc0OG7YadZdvm7KPQ1j0ehmQQHckgEWOfgpptzcnWgESovxudpTA=="
+        },
         "jquery": {
             "version": "3.3.1",
             "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.3.1.tgz",
@@ -11261,13 +11348,9 @@
             }
         },
         "node-fetch": {
-            "version": "1.7.3",
+            "version": "2.3.0",
-            "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz",
+            "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.3.0.tgz",
-            "integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==",
+            "integrity": "sha512-MOd8pV3fxENbryESLgVIeaGKrdl+uaYhCSSVkjeOb/31/njTpcis5aWfdqgNlHIrKOLRbMnfPINPOML2CIFeXA=="
-            "requires": {
-                "encoding": "^0.1.11",
-                "is-stream": "^1.0.1"
-            }
         },
         "node-forge": {
             "version": "0.7.5",
@@ -16377,6 +16460,11 @@
                 }
             }
         },
+        "seedrandom": {
+            "version": "2.4.3",
+            "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-2.4.3.tgz",
+            "integrity": "sha1-JDhQTa0zkXMUv/GKxNeU8W1qrsw="
+        },
         "select-hose": {
             "version": "2.0.0",
             "resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz",

package.json

@@ -16,6 +16,7 @@
     "main": "build/main.js",
     "dependencies": {
         "@azure/storage-blob": "^10.3.0",
+        "@tensorflow/tfjs": "^1.0.3",
         "@types/snapsvg": "^0.4.35",
         "axios": "^0.18.0",
         "bootstrap": "^4.1.3",
@@ -23,8 +24,10 @@
         "crypto-js": "^3.1.9-1",
         "dotenv": "^7.0.0",
         "google-protobuf": "^3.6.1",
+        "jpeg-js": "^0.3.4",
         "lodash": "^4.17.11",
         "md5.js": "^1.3.5",
+        "node-fetch": "^2.3.0",
         "node-int64": "^0.4.0",
         "rc-align": "^2.4.5",
         "rc-checkbox": "^2.1.6",

src/common/htmlFileReader.test.ts

@@ -10,17 +10,7 @@ describe("Html File Reader", () => {
 
     beforeEach(() => {
         assetTestCache.clear();
-
+        MockFactory.mockElement(assetTestCache);
-        document.createElement = jest.fn((elementType) => {
-            switch (elementType) {
-                case "img":
-                    return mockImage();
-                case "video":
-                    return mockVideo();
-                case "canvas":
-                    return mockCanvas();
-            }
-        });
     });
 
     it("Resolves promise after successfully reading file", async () => {
@@ -234,67 +224,4 @@ describe("Html File Reader", () => {
             await expect(HtmlFileReader.getAssetFrameImage(videoErrorFrame)).rejects.not.toBeNull();
         });
     });
-
-    const mockImage = jest.fn(() => {
-        const element: any = {
-            naturalWidth: 0,
-            naturalHeight: 0,
-            onload: jest.fn(),
-        };
-
-        setImmediate(() => {
-            const asset = assetTestCache.get(element.src);
-            element.naturalWidth = asset.size.width;
-            element.naturalHeight = asset.size.height;
-
-            element.onload();
-        });
-
-        return element;
-    });
-
-    const mockVideo = jest.fn(() => {
-        const element: any = {
-            src: "",
-            duration: 0,
-            currentTime: 0,
-            videoWidth: 0,
-            videoHeight: 0,
-            onloadedmetadata: jest.fn(),
-            onseeked: jest.fn(),
-            onerror: jest.fn(),
-        };
-
-        setImmediate(() => {
-            const asset = assetTestCache.get(element.src);
-            if (asset.name.toLowerCase().indexOf("error") > -1) {
-                element.onerror("An error occurred loading the video");
-            } else {
-                element.videoWidth = asset.size.width;
-                element.videoHeight = asset.size.height;
-                element.currentTime = asset.timestamp;
-                element.onloadedmetadata();
-                element.onseeked();
-            }
-        });
-
-        return element;
-    });
-
-    const mockCanvas = jest.fn(() => {
-        const canvas: any = {
-            width: 0,
-            height: 0,
-            getContext: jest.fn(() => {
-                return {
-                    drawImage: jest.fn(),
-                };
-            }),
-            toBlob: jest.fn((callback) => {
-                callback(new Blob(["Binary image data"]));
-            }),
-        };
-
-        return canvas;
-    });
 });

src/common/localization/en-us.ts

@@ -143,6 +143,7 @@ export const english: IAppStrings = {
         warnings: {
             existingName: "Tag name already exists. Choose another name",
             emptyName: "Cannot have an empty tag name",
+            unknownTagName: "Unknown",
         },
         toolbar: {
             add: "Add new tag",
@@ -231,6 +232,7 @@ export const english: IAppStrings = {
             nextAsset: "Next Asset",
             saveProject: "Save Project",
             exportProject: "Export Project",
+            activeLearning: "Active Learning",
         },
         videoPlayer: {
             previousTaggedFrame: {
@@ -275,9 +277,8 @@ export const english: IAppStrings = {
         messages: {
             enforceTaggedRegions: {
                 title: "Invalid region(s) detected",
-                // tslint:disable-next-line:max-line-length
                 description: "1 or more regions have not been tagged.  Ensure all regions are tagged before \
-                continuing to next asset.",
+                    continuing to next asset.",
             },
         },
     },
@@ -391,6 +392,40 @@ export const english: IAppStrings = {
     },
     activeLearning: {
         title: "Active Learning",
+        form: {
+            properties: {
+                modelPathType: {
+                    title: "Model Provider",
+                    description: "Where to load the training model from",
+                    options: {
+                        preTrained: "Pre-trained Coco SSD",
+                        customFilePath: "Custom (File path)",
+                        customWebUrl: "Custom (Url)",
+                    },
+                },
+                autoDetect: {
+                    title: "Auto Detect",
+                    description: "Whether or not to automatically make predictions as you navigate between assets",
+                },
+                modelPath: {
+                    title: "Model path",
+                    description: "Select a model from your local file system",
+                },
+                modelUrl: {
+                    title: "Model URL",
+                    description: "Load your model from a public web URL",
+                },
+                predictTag: {
+                    title: "Predict Tag",
+                    description: "Whether or not to automatically include tags in predictions",
+                },
+            },
+        },
+        messages: {
+            loadingModel: "Loading active learning model...",
+            errorLoadModel: "Error loading active learning model",
+            saveSuccess: "Successfully saved active learning settings",
+        },
     },
     profile: {
         settings: "Profile Settings",
@@ -444,5 +479,10 @@ export const english: IAppStrings = {
             title: "Error exporting project",
             message: "Project is missing export format.  Please select an export format in the export setting page.",
         },
+        activeLearningPredictionError: {
+            title: "Active Learning Error",
+            message: "An error occurred while predicting regions in the current asset. \
+                Please verify your active learning configuration and try again",
+        },
     },
 };

src/common/localization/es-cl.ts

@@ -60,8 +60,8 @@ export const spanish: IAppStrings = {
         },
         securityTokens: {
             title: "Tokens de seguridad",
-            // tslint:disable-next-line:max-line-length
+            description: "Los tokens de seguridad se utilizan para cifrar datos confidenciales \
-            description: "Los tokens de seguridad se utilizan para cifrar datos confidenciales dentro de la configuración del proyecto",
+                dentro de la configuración del proyecto",
         },
         version: {
             description: "Versión:",
@@ -144,6 +144,7 @@ export const spanish: IAppStrings = {
         warnings: {
             existingName: "Nombre de etiqueta ya existe. Elige otro nombre",
             emptyName: "El nombre de etiqueta no puede ser vacío",
+            unknownTagName: "Desconocido",
         },
         toolbar: {
             add: "Agregar nueva etiqueta",
@@ -233,6 +234,7 @@ export const spanish: IAppStrings = {
             nextAsset: "Siguiente activo",
             saveProject: "Guardar Proyecto",
             exportProject: "Exprtar Proyecto",
+            activeLearning: "Aprendizaje Activo",
         },
         videoPlayer: {
             previousTaggedFrame: {
@@ -278,8 +280,8 @@ export const spanish: IAppStrings = {
         messages: {
             enforceTaggedRegions: {
                 title: "Las regiones no válidas detectadas",
-                // tslint:disable-next-line:max-line-length
+                description: "1 o más regiones no se han etiquetado.  \
-                description: "1 o más regiones no se han etiquetado.  Por favor, etiquete todas las regiones antes de continuar con el siguiente activo.",
+                    Por favor, etiquete todas las regiones antes de continuar con el siguiente activo.",
             },
         },
     },
@@ -393,6 +395,41 @@ export const spanish: IAppStrings = {
     },
     activeLearning: {
         title: "Aprendizaje Activo",
+        form: {
+            properties: {
+                modelPathType: {
+                    title: "Proveedor del modelo",
+                    description: "Fuente desde la cual cargar el modelo",
+                    options: {
+                        preTrained: "SSD de coco pre-entrenado",
+                        customFilePath: "Personalizado (ruta de archivo)",
+                        customWebUrl: "Personalizado (URL)",
+                    },
+                },
+                autoDetect: {
+                    title: "Detección automática",
+                    description: "Si desea o no realizar automáticamente predicciones a \
+                        medida que navega entre activos",
+                },
+                modelPath: {
+                    title: "Ruta de modelo",
+                    description: "Seleccione un modelo de su sistema de archivos local",
+                },
+                modelUrl: {
+                    title: "URL del modelo",
+                    description: "Cargue el modelo desde una URL web pública",
+                },
+                predictTag: {
+                    title: "Predecir etiqueta",
+                    description: "Si se incluirán o no automáticamente las etiquetas en las predicciones",
+                },
+            },
+        },
+        messages: {
+            loadingModel: "Cargando modelo...",
+            errorLoadModel: "Error al cargar el modelo",
+            saveSuccess: "La configuración de aprendizaje activa se ha guardada correctamente",
+        },
     },
     profile: {
         settings: "Configuración de Perfíl",
@@ -448,5 +485,10 @@ export const spanish: IAppStrings = {
             message: `Proyecto falta el formato de exportación. Seleccione un formato de exportación en la página
             de configuración de exportación.`,
         },
+        activeLearningPredictionError: {
+            title: "Error de aprendizaje",
+            message: "Se ha producido un error al predecir regiones en el activo actual. \
+                Compruebe la configuración de aprendizaje activa y vuelva a intentarlo",
+        },
     },
 };

src/common/mockFactory.ts

@@ -3,7 +3,7 @@ import {
     AssetState, AssetType, IApplicationState, IAppSettings, IAsset, IAssetMetadata,
     IConnection, IExportFormat, IProject, ITag, StorageType, ISecurityToken,
     EditorMode, IAppError, IProjectVideoSettings, ErrorCode,
-    IPoint, IRegion, RegionType,
+    IPoint, IRegion, RegionType, ModelPathType,
 } from "../models/applicationState";
 import { IV1Project, IV1Region } from "../models/v1Models";
 import { ExportAssetState } from "../providers/export/exportProvider";
@@ -33,6 +33,7 @@ import { SelectionMode } from "vott-ct/lib/js/CanvasTools/Interface/ISelectorSet
 import { IKeyboardBindingProps } from "../react/components/common/keyboardBinding/keyboardBinding";
 import { KeyEventType } from "../react/components/common/keyboardManager/keyboardManager";
 import { IKeyboardRegistrations } from "../react/components/common/keyboardManager/keyboardRegistrationManager";
+import { IActiveLearningPageProps } from "../react/components/pages/activeLearning/activeLearningPage";
 
 export default class MockFactory {
 
@@ -283,6 +284,13 @@ export default class MockFactory {
             targetConnection: connection,
             tags: MockFactory.createTestTags(tagCount),
             videoSettings: MockFactory.createVideoSettings(),
+            activeLearningSettings: {
+                modelPathType: ModelPathType.Coco,
+                modelPath: "",
+                modelUrl: "",
+                autoDetect: false,
+                predictTag: false,
+            },
             autoSave: true,
         };
     }
@@ -886,6 +894,21 @@ export default class MockFactory {
         };
     }
 
+    /**
+     * Creates fake IActiveLearningPageProps
+     * @param projectId Current project ID
+     */
+    public static activeLearningProps(projectId?: string): IActiveLearningPageProps {
+        return {
+            actions: (projectActions as any) as IProjectActions,
+            history: MockFactory.history(),
+            location: MockFactory.location(),
+            match: MockFactory.match(projectId, "active-learning"),
+            project: null,
+            recentProjects: MockFactory.createTestProjects(),
+        };
+    }
+
     /**
      * Creates fake IEditorPageProps
      * @param projectId Current project ID
@@ -1012,6 +1035,93 @@ export default class MockFactory {
         };
     }
 
+    public static mockElement(assetTestCache: Map<string, IAsset>) {
+        document.createElement = jest.fn((elementType) => {
+            switch (elementType) {
+                case "img":
+                    const mockImage = MockFactory.mockImage(assetTestCache);
+                    return mockImage();
+                case "video":
+                    const mockVideo = MockFactory.mockVideo(assetTestCache);
+                    return mockVideo();
+                case "canvas":
+                    const mockCanvas = MockFactory.mockCanvas();
+                    return mockCanvas();
+            }
+        });
+    }
+
+    public static mockImage(assetTestCache: Map<string, IAsset>) {
+        return jest.fn(() => {
+            const element: any = {
+                naturalWidth: 0,
+                naturalHeight: 0,
+                onload: jest.fn(),
+            };
+
+            setImmediate(() => {
+                const asset = assetTestCache.get(element.src);
+                if (asset) {
+                    element.naturalWidth = asset.size.width;
+                    element.naturalHeight = asset.size.height;
+                }
+
+                element.onload();
+            });
+
+            return element;
+        });
+    }
+
+    public static mockVideo(assetTestCache: Map<string, IAsset>) {
+        return jest.fn(() => {
+            const element: any = {
+                src: "",
+                duration: 0,
+                currentTime: 0,
+                videoWidth: 0,
+                videoHeight: 0,
+                onloadedmetadata: jest.fn(),
+                onseeked: jest.fn(),
+                onerror: jest.fn(),
+            };
+
+            setImmediate(() => {
+                const asset = assetTestCache.get(element.src);
+                if (asset.name.toLowerCase().indexOf("error") > -1) {
+                    element.onerror("An error occurred loading the video");
+                } else {
+                    element.videoWidth = asset.size.width;
+                    element.videoHeight = asset.size.height;
+                    element.currentTime = asset.timestamp;
+                    element.onloadedmetadata();
+                    element.onseeked();
+                }
+            });
+
+            return element;
+        });
+    }
+
+    public static mockCanvas() {
+        return jest.fn(() => {
+            const canvas: any = {
+                width: 800,
+                height: 600,
+                getContext: jest.fn(() => {
+                    return {
+                        drawImage: jest.fn(),
+                    };
+                }),
+                toBlob: jest.fn((callback) => {
+                    callback(new Blob(["Binary image data"]));
+                }),
+            };
+
+            return canvas;
+        });
+    }
+
     private static pageProps(projectId: string, method: string) {
         return {
             project: null,
@@ -1093,5 +1203,4 @@ export default class MockFactory {
                 return StorageType.Other;
         }
     }
-
 }

src/common/strings.ts

@@ -154,6 +154,7 @@ export interface IAppStrings {
         warnings: {
             existingName: string;
             emptyName: string;
+            unknownTagName: string;
         }
     };
     connections: {
@@ -230,6 +231,7 @@ export interface IAppStrings {
             nextAsset: string;
             saveProject: string;
             exportProject: string;
+            activeLearning: string;
         }
         videoPlayer: {
             nextTaggedFrame: {
@@ -387,6 +389,40 @@ export interface IAppStrings {
     };
     activeLearning: {
         title: string;
+        form: {
+            properties: {
+                modelPathType: {
+                    title: string,
+                    description: string,
+                    options: {
+                        preTrained: string,
+                        customFilePath: string,
+                        customWebUrl: string,
+                    },
+                },
+                autoDetect: {
+                    title: string,
+                    description: string,
+                },
+                predictTag: {
+                    title: string,
+                    description: string,
+                },
+                modelPath: {
+                    title: string,
+                    description: string,
+                },
+                modelUrl: {
+                    title: string,
+                    description: string,
+                },
+            },
+        }
+        messages: {
+            loadingModel: string;
+            errorLoadModel: string;
+            saveSuccess: string;
+        }
     };
     profile: {
         settings: string;
@@ -403,6 +439,7 @@ export interface IAppStrings {
         importError: IErrorMetadata,
         pasteRegionTooBigError: IErrorMetadata,
         exportFormatNotFound: IErrorMetadata,
+        activeLearningPredictionError: IErrorMetadata,
     };
 }
 

src/models/applicationState.ts

@@ -1,4 +1,5 @@
 import { ExportAssetState } from "../providers/export/exportProvider";
+import { IAssetPreviewSettings } from "../react/components/common/assetPreview/assetPreview";
 
 /**
  * @name - Application State
@@ -49,6 +50,7 @@ export enum ErrorCode {
     ExportFormatNotFound = "exportFormatNotFound",
     PasteRegionTooBig = "pasteRegionTooBig",
     OverloadedKeyBinding = "overloadedKeyBinding",
+    ActiveLearningPredictionError = "activeLearningPredictionError",
 }
 
 /**
@@ -112,6 +114,7 @@ export interface IProject {
     targetConnection: IConnection;
     exportFormat: IExportFormat;
     videoSettings: IProjectVideoSettings;
+    activeLearningSettings: IActiveLearningSettings;
     autoSave: boolean;
     assets?: { [index: string]: IAsset };
     lastVisitedAssetId?: string;
@@ -198,6 +201,44 @@ export interface IProjectVideoSettings {
     frameExtractionRate: number;
 }
 
+/**
+ * @name - Model Path Type
+ * @description - Defines the mechanism to load the TF.js model for Active Learning
+ * @member Coco - Specifies the default/generic pre-trained Coco-SSD model
+ * @member File - Specifies to load a custom model from filesystem
+ * @member Url - Specifies to load a custom model from a web server
+ */
+export enum ModelPathType {
+    Coco = "coco",
+    File = "file",
+    Url = "url",
+}
+
+/**
+ * Properties for additional project settings
+ * @member activeLearningSettings - Active Learning settings
+ */
+export interface IAdditionalPageSettings extends IAssetPreviewSettings {
+    activeLearningSettings: IActiveLearningSettings;
+}
+
+/**
+ * @name - Active Learning Settings for the project
+ * @description - Defines the active learning settings within a VoTT project
+ * @member modelPathType - Model loading type ["coco", "file", "url"]
+ * @member modelPath - Local filesystem path to the TF.js model
+ * @member modelUrl - Web url to the TF.js model
+ * @member autoDetect - Flag for automatically call the model while opening a new asset
+ * @member predictTag - Flag to predict also the tag name other than the rectangle coordinates only
+ */
+export interface IActiveLearningSettings {
+    modelPathType: ModelPathType;
+    modelPath?: string;
+    modelUrl?: string;
+    autoDetect: boolean;
+    predictTag: boolean;
+}
+
 /**
  * @name - Asset Video Settings
  * @description - Defines the settings for video assets
@@ -231,6 +272,7 @@ export interface IAsset {
     format?: string;
     timestamp?: number;
     parent?: IAsset;
+    predicted?: boolean;
 }
 
 /**

src/providers/activeLearning/electronProxyHandler.test.ts

@@ -0,0 +1,33 @@
+jest.mock("../storage/localFileSystemProxy");
+import { LocalFileSystemProxy } from "../storage/localFileSystemProxy";
+import { ElectronProxyHandler } from "./electronProxyHandler";
+import * as tf from "@tensorflow/tfjs";
+// tslint:disable-next-line:no-var-requires
+const modelJson = require("../../../cocoSSDModel/model.json");
+
+describe("Load default model from filesystem with TF io.IOHandler", () => {
+    it("Check file system proxy is correctly called", async () => {
+        const storageProviderMock = LocalFileSystemProxy as jest.Mock<LocalFileSystemProxy>;
+        storageProviderMock.mockClear();
+
+        storageProviderMock.prototype.readText = jest.fn((fileName) => {
+            return Promise.resolve(JSON.stringify(modelJson));
+        });
+
+        storageProviderMock.prototype.readBinary = jest.fn((fileName) => {
+            return Promise.resolve([]);
+        });
+
+        const handler = new ElectronProxyHandler("folder");
+        try {
+            const model = await tf.loadGraphModel(handler);
+        } catch (_) {
+            // fully loading TF model fails as it has to load also weights
+        }
+
+        expect(LocalFileSystemProxy.prototype.readText).toBeCalledWith("/model.json");
+
+        // Coco SSD Lite default embedded model has 5 weights matrix
+        expect(LocalFileSystemProxy.prototype.readBinary).toBeCalledTimes(5);
+    });
+});

src/providers/activeLearning/electronProxyHandler.ts

@@ -0,0 +1,76 @@
+import * as tfc from "@tensorflow/tfjs-core";
+import { LocalFileSystemProxy, ILocalFileSystemProxyOptions } from "../../providers/storage/localFileSystemProxy";
+
+export class ElectronProxyHandler implements tfc.io.IOHandler {
+    protected readonly provider: LocalFileSystemProxy;
+
+    constructor(folderPath: string) {
+        const options: ILocalFileSystemProxyOptions = { folderPath };
+        this.provider = new LocalFileSystemProxy(options);
+    }
+
+    public async load(): Promise<tfc.io.ModelArtifacts> {
+        const modelJSON = JSON.parse(await this.provider.readText("/model.json"));
+
+        const modelArtifacts: tfc.io.ModelArtifacts = {
+            modelTopology: modelJSON.modelTopology,
+        };
+
+        if (modelJSON.weightsManifest != null) {
+            const [weightSpecs, weightData] =
+                await this.loadWeights(modelJSON.weightsManifest);
+            modelArtifacts.weightSpecs = weightSpecs;
+            modelArtifacts.weightData = weightData;
+        }
+
+        return modelArtifacts;
+    }
+
+    public async loadClasses(): Promise<JSON> {
+        const json = await this.provider.readText("/classes.json");
+        return json ? JSON.parse(json) : null;
+    }
+
+    private async loadWeights(weightsManifest: tfc.io.WeightsManifestConfig)
+        : Promise<[tfc.io.WeightsManifestEntry[], ArrayBuffer]> {
+        const buffers: Buffer[] = [];
+        const weightSpecs: tfc.io.WeightsManifestEntry[] = [];
+
+        for (const group of weightsManifest) {
+            for (const shardName of group.paths) {
+                const buffer = await this.provider.readBinary("/" + shardName);
+                buffers.push(buffer);
+            }
+            weightSpecs.push(...group.weights);
+        }
+
+        return [weightSpecs, this.toArrayBuffer(buffers)];
+    }
+
+    /**
+     * Convert a Buffer or an Array of Buffers to an ArrayBuffer.
+     *
+     * If the input is an Array of Buffers, they will be concatenated in the
+     * specified order to form the output ArrayBuffer.
+     */
+    private toArrayBuffer(buf: Buffer | Buffer[]): ArrayBuffer {
+        if (Array.isArray(buf)) {
+            // An Array of Buffers.
+            let totalLength = 0;
+            for (const buffer of buf) {
+                totalLength += buffer.length;
+            }
+
+            const ab = new ArrayBuffer(totalLength);
+            const view = new Uint8Array(ab);
+            let pos = 0;
+            for (const buffer of buf) {
+                pos += buffer.copy(view, pos);
+            }
+            return ab;
+        } else {
+            // A single Buffer. Return a copy of the underlying ArrayBuffer slice.
+            return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
+        }
+    }
+}

src/providers/activeLearning/objectDetection.test.ts

@@ -0,0 +1,123 @@
+import * as tf from "@tensorflow/tfjs";
+jest.mock("../storage/localFileSystemProxy");
+import { LocalFileSystemProxy } from "../storage/localFileSystemProxy";
+import { ObjectDetection, DetectedObject } from "./objectDetection";
+import { strings } from "../../common/strings";
+// tslint:disable-next-line:no-var-requires
+const modelJson = require("../../../cocoSSDModel/model.json");
+
+describe("Load an Object Detection model", () => {
+    it("Load model from file system using proxy", async () => {
+        const storageProviderMock = LocalFileSystemProxy as jest.Mock<LocalFileSystemProxy>;
+        storageProviderMock.mockClear();
+
+        storageProviderMock.prototype.readText = jest.fn((fileName) => {
+            return Promise.resolve(JSON.stringify(modelJson));
+        });
+
+        storageProviderMock.prototype.readBinary = jest.fn((fileName) => {
+            return Promise.resolve([]);
+        });
+
+        const model = new ObjectDetection();
+
+        try {
+            await model.load("path");
+        } catch (_) {
+            // fully loading TF model fails has it has to load also weights
+        }
+
+        expect(LocalFileSystemProxy.prototype.readText).toBeCalledWith("/model.json");
+
+        // Coco SSD Lite default embedded model has 5 weights matrix
+        expect(LocalFileSystemProxy.prototype.readBinary).toBeCalledTimes(5);
+
+        // Modal not properly loaded as readBinary mock is not really loading the weights
+        expect(model.loaded).toBeFalsy();
+
+        const noDetection = await model.detect(null);
+        expect(noDetection.length).toEqual(0);
+
+        model.dispose();
+    });
+
+    it("Load model from http url", async () => {
+        window.fetch = jest.fn().mockImplementation((url, o) => {
+            if (url === "http://url/model.json") {
+                return Promise.resolve({
+                    ok: true,
+                    json: () => modelJson,
+                });
+            } else {
+                return Promise.resolve({
+                    ok: true,
+                    data: () => [],
+                });
+            }
+        });
+
+        const model = new ObjectDetection();
+
+        expect(model.load("http://url")).rejects.not.toBeNull();
+        expect(window.fetch).toBeCalledTimes(1);
+
+        // Modal not properly loaded as readBinary mock is not really loading the weights
+        expect(model.loaded).toBeFalsy();
+
+        const noDetection = await model.detect(null);
+        expect(noDetection.length).toEqual(0);
+
+        model.dispose();
+    });
+});
+
+describe("Test Detection on Fake Model", () => {
+    beforeEach(() => {
+        spyOn(tf, "loadGraphModel").and.callFake(() => {
+            const model = {
+                executeAsync:
+                    (x: tf.Tensor) => [tf.ones([1, 1917, 90]), tf.ones([1, 1917, 1, 4])],
+            };
+
+            return model;
+        });
+    });
+
+    it("ObjectDetection detect method should generate output", async () => {
+        const model = new ObjectDetection();
+        await model.load("path");
+
+        const x = tf.zeros([227, 227, 3]) as tf.Tensor3D;
+
+        const data = await model.detect(x, 1);
+
+        expect(data).toEqual([{bbox: [227, 227, 0, 0], class: strings.tags.warnings.unknownTagName, score: 1}]);
+    });
+});
+
+describe("Test predictImage on Fake Model", () => {
+    beforeEach(() => {
+        spyOn(tf, "loadGraphModel").and.callFake(() => {
+            const model = {
+                executeAsync:
+                    (x: tf.Tensor) => [tf.ones([1, 1917, 90]), tf.ones([1, 1917, 1, 4])],
+            };
+
+            return model;
+        });
+    });
+
+    it("predictImage on a fake image", async () => {
+        const model = new ObjectDetection();
+        await model.load("path");
+
+        const x = tf.zeros([227, 227, 3]) as tf.Tensor3D;
+        const regions = await model.predictImage(x, false, 1, 1);
+
+        expect(regions.length).toEqual(20);
+        expect(regions[0].boundingBox.left).toEqual(227);
+        expect(regions[0].boundingBox.top).toEqual(227);
+        expect(regions[0].boundingBox.width).toEqual(0);
+        expect(regions[0].boundingBox.height).toEqual(0);
+    });
+});

src/providers/activeLearning/objectDetection.ts

@@ -0,0 +1,253 @@
+import axios from "axios";
+import * as shortid from "shortid";
+import * as tf from "@tensorflow/tfjs";
+import { ElectronProxyHandler } from "./electronProxyHandler";
+import { IRegion, RegionType } from "../../models/applicationState";
+import { strings } from "../../common/strings";
+
+// tslint:disable-next-line:interface-over-type-literal
+export type DetectedObject = {
+    bbox: [number, number, number, number];  // [x, y, width, height]
+    class: string;
+    score: number;
+};
+
+/**
+ * Defines supported data types supported by Tensorflow JS
+ */
+export type ImageObject = tf.Tensor3D | ImageData | HTMLImageElement | HTMLCanvasElement | HTMLVideoElement;
+
+/**
+ * Object Dectection loads active learning models and predicts regions
+ */
+export class ObjectDetection {
+    private modelLoaded: boolean = false;
+
+    get loaded(): boolean {
+        return this.modelLoaded;
+    }
+
+    private model: tf.GraphModel;
+    private jsonClasses: JSON;
+
+    /**
+     * Dispose the tensors allocated by the model. You should call this when you
+     * are done with the model.
+     */
+    public dispose() {
+        if (this.model) {
+            this.model.dispose();
+        }
+    }
+
+    /**
+     * Load a TensorFlow.js Object Detection model from file: or http URL.
+     * @param modelFolderPath file: or http URL to the model
+     */
+    public async load(modelFolderPath: string) {
+        try {
+            if (modelFolderPath.toLowerCase().startsWith("http://") ||
+                modelFolderPath.toLowerCase().startsWith("https://")) {
+                this.model = await tf.loadGraphModel(modelFolderPath + "/model.json");
+
+                const response = await axios.get(modelFolderPath + "/classes.json");
+                this.jsonClasses = JSON.parse(JSON.stringify(response.data));
+            } else {
+                const handler = new ElectronProxyHandler(modelFolderPath);
+                this.model = await tf.loadGraphModel(handler);
+                this.jsonClasses = await handler.loadClasses();
+            }
+
+            // Warmup the model.
+            const result = await this.model.executeAsync(tf.zeros([1, 300, 300, 3])) as tf.Tensor[];
+            result.forEach(async (t) => await t.data());
+            result.forEach(async (t) => t.dispose());
+            this.modelLoaded = true;
+        } catch (err) {
+            this.modelLoaded = false;
+            throw err;
+        }
+    }
+
+    /**
+     * Predict Regions from an HTMLImageElement returning list of IRegion.
+     * @param image ImageObject to be used for prediction
+     * @param predictTag Flag indicates if predict only region bounding box of tag too.
+     * @param xRatio Width compression ratio between the HTMLImageElement and the original image.
+     * @param yRatio Height compression ratio between the HTMLImageElement and the original image.
+     */
+    public async predictImage(image: ImageObject, predictTag: boolean, xRatio: number, yRatio: number)
+        : Promise<IRegion[]> {
+        const regions: IRegion[] = [];
+
+        const predictions = await this.detect(image);
+        predictions.forEach((prediction) => {
+            const left = Math.max(0, prediction.bbox[0] * xRatio);
+            const top = Math.max(0, prediction.bbox[1] * yRatio);
+            const width = Math.max(0, prediction.bbox[2] * xRatio);
+            const height = Math.max(0, prediction.bbox[3] * yRatio);
+
+            regions.push({
+                id: shortid.generate(),
+                type: RegionType.Rectangle,
+                tags: predictTag ? [prediction.class] : [],
+                boundingBox: {
+                    left,
+                    top,
+                    width,
+                    height,
+                },
+                points: [{
+                    x: left,
+                    y: top,
+                },
+                {
+                    x: left + width,
+                    y: top,
+                },
+                {
+                    x: left + width,
+                    y: top + height,
+                },
+                {
+                    x: left,
+                    y: top + height,
+                }],
+            });
+        });
+
+        return regions;
+    }
+
+    /**
+     * Detect objects for an image returning a list of bounding boxes with
+     * associated class and score.
+     *
+     * @param img The image to detect objects from. Can be a tensor or a DOM
+     *     element image, video, or canvas.
+     * @param maxNumBoxes The maximum number of bounding boxes of detected
+     * objects. There can be multiple objects of the same class, but at different
+     * locations. Defaults to 20.
+     *
+     */
+    public async detect(img: ImageObject, maxNumBoxes: number = 20): Promise<DetectedObject[]> {
+        if (this.model) {
+            return this.infer(img, maxNumBoxes);
+        }
+
+        return [];
+    }
+
+    /**
+     * Infers through the model.
+     *
+     * @param img The image to classify. Can be a tensor or a DOM element image,
+     * video, or canvas.
+     * @param maxNumBoxes The maximum number of bounding boxes of detected
+     * objects. There can be multiple objects of the same class, but at different
+     * locations. Defaults to 20.
+     */
+    private async infer(img: ImageObject, maxNumBoxes: number = 20): Promise<DetectedObject[]> {
+        const batched = tf.tidy(() => {
+            if (!(img instanceof tf.Tensor)) {
+                img = tf.browser.fromPixels(img);
+            }
+            // Reshape to a single-element batch so we can pass it to executeAsync.
+            return img.expandDims(0);
+        });
+        const height = batched.shape[1];
+        const width = batched.shape[2];
+
+        // model returns two tensors:
+        // 1. box classification score with shape of [1, 1917, 90]
+        // 2. box location with shape of [1, 1917, 1, 4]
+        // where 1917 is the number of box detectors, 90 is the number of classes.
+        // and 4 is the four coordinates of the box.
+        const result = await this.model.executeAsync(batched) as tf.Tensor[];
+
+        const scores = result[0].dataSync() as Float32Array;
+        const boxes = result[1].dataSync() as Float32Array;
+
+        // clean the webgl tensors
+        batched.dispose();
+        tf.dispose(result);
+
+        const [maxScores, classes] = this.calculateMaxScores(scores, result[0].shape[1], result[0].shape[2]);
+
+        const prevBackend = tf.getBackend();
+        // run post process in cpu
+        tf.setBackend("cpu");
+        const indexTensor = tf.tidy(() => {
+            const boxes2 = tf.tensor2d(boxes, [result[1].shape[1], result[1].shape[3]]);
+            return tf.image.nonMaxSuppression(boxes2, maxScores, maxNumBoxes, 0.5, 0.5);
+        });
+
+        const indexes = indexTensor.dataSync() as Float32Array;
+        indexTensor.dispose();
+
+        // restore previous backend
+        tf.setBackend(prevBackend);
+
+        return this.buildDetectedObjects(width, height, boxes, maxScores, indexes, classes);
+    }
+
+    private buildDetectedObjects(
+        width: number, height: number, boxes: Float32Array, scores: number[],
+        indexes: Float32Array, classes: number[]): DetectedObject[] {
+        const count = indexes.length;
+        const objects: DetectedObject[] = [];
+
+        for (let i = 0; i < count; i++) {
+            const bbox = [];
+            for (let j = 0; j < 4; j++) {
+                bbox[j] = boxes[indexes[i] * 4 + j];
+            }
+            const minY = bbox[0] * height;
+            const minX = bbox[1] * width;
+            const maxY = bbox[2] * height;
+            const maxX = bbox[3] * width;
+            bbox[0] = minX;
+            bbox[1] = minY;
+            bbox[2] = maxX - minX;
+            bbox[3] = maxY - minY;
+            objects.push({
+                bbox: bbox as [number, number, number, number],
+                class: this.getClass(i, indexes, classes),
+                score: scores[indexes[i]],
+            });
+        }
+
+        return objects;
+    }
+
+    private getClass(index: number, indexes: Float32Array, classes: number[]): string {
+        if (this.jsonClasses && index < indexes.length && indexes[index] < classes.length) {
+            const classId = classes[indexes[index]] - 1;
+            const classObject = this.jsonClasses[classId];
+
+            return classObject ? classObject.displayName : strings.tags.warnings.unknownTagName;
+        }
+
+        return "";
+    }
+
+    private calculateMaxScores(
+        scores: Float32Array, numBoxes: number,
+        numClasses: number): [number[], number[]] {
+        const maxes = [];
+        const classes = [];
+        for (let i = 0; i < numBoxes; i++) {
+            let max = Number.MIN_VALUE;
+            let index = -1;
+            for (let j = 0; j < numClasses; j++) {
+                if (scores[i * numClasses + j] > max) {
+                    max = scores[i * numClasses + j];
+                    index = j;
+                }
+            }
+            maxes[i] = max;
+            classes[i] = index;
+        }
+        return [maxes, classes];
+    }
+}

src/react/components/common/assetPreview/imageAsset.tsx

@@ -12,7 +12,8 @@ export class ImageAsset extends React.Component<IAssetProps> {
             <img ref={this.image}
                 src={this.props.asset.path}
                 onLoad={this.onLoad}
-                onError={this.props.onError} />);
+                onError={this.props.onError}
+                crossOrigin="anonymous" />);
     }
 
     private onLoad = () => {

src/react/components/common/assetPreview/tfrecordAsset.tsx

@@ -31,7 +31,8 @@ export class TFRecordAsset extends React.Component<IAssetProps, ITFRecordState>
             <img ref={this.image}
                 src={this.state.tfRecordImage64}
                 onLoad={this.onLoad}
-                onError={this.onError} />
+                onError={this.onError}
+                crossOrigin="anonymous" />
         );
     }
 

src/react/components/common/assetPreview/videoAsset.tsx

@@ -73,7 +73,8 @@ export class VideoAsset extends React.Component<IVideoAssetProps> {
                 height="100%"
                 autoPlay={autoPlay}
                 src={videoPath}
-                onError={this.props.onError}>
+                onError={this.props.onError}
+                crossOrigin="anonymous">
                 <BigPlayButton position="center" />
                 {autoPlay &&
                     <ControlBar autoHide={false}>

src/react/components/common/externalPicker/externalPicker.test.tsx

@@ -5,9 +5,7 @@ import ExternalPicker, { IExternalPickerProps, IExternalPickerState, FilterOpera
 import MockFactory from "../../../../common/mockFactory";
 
 describe("External Picker", () => {
-    const onChangeHandler = jest.fn(() => {
+    const onChangeHandler = jest.fn();
-        console.log("hi");
-    });
     const defaultProps = createProps({
         id: "my-custom-control",
         value: "",

src/react/components/pages/activeLearning/activeLearningForm.json

@@ -0,0 +1,82 @@
+{
+    "type": "object",
+    "properties": {
+        "modelPathType": {
+            "type": "string",
+            "title": "${strings.activeLearning.form.properties.modelPathType.title}",
+            "description": "${strings.activeLearning.form.properties.modelPathType.description}",
+            "enum": [
+                "coco",
+                "file",
+                "url"
+            ],
+            "default": "coco",
+            "enumNames": [
+                "${strings.activeLearning.form.properties.modelPathType.options.preTrained}",
+                "${strings.activeLearning.form.properties.modelPathType.options.customFilePath}",
+                "${strings.activeLearning.form.properties.modelPathType.options.customWebUrl}"
+            ]
+        },
+        "autoDetect": {
+            "title": "${strings.activeLearning.form.properties.autoDetect.title}",
+            "description": "${strings.activeLearning.form.properties.autoDetect.description}",
+            "type": "boolean"
+        },
+        "predictTag": {
+            "title": " ${strings.activeLearning.form.properties.predictTag.title}",
+            "description": "${strings.activeLearning.form.properties.predictTag.description}",
+            "type": "boolean"
+        }
+    },
+    "dependencies": {
+        "modelPathType": {
+            "oneOf": [
+                {
+                    "properties": {
+                        "modelPathType": {
+                            "enum": [
+                                "coco"
+                            ]
+                        }
+                    }
+                },
+                {
+                    "required": [
+                        "modelPath"
+                    ],
+                    "properties": {
+                        "modelPathType": {
+                            "enum": [
+                                "file"
+                            ]
+                        },
+                        "modelPath": {
+                            "title": "${strings.activeLearning.form.properties.modelPath.title}",
+                            "description": "${strings.activeLearning.form.properties.modelPath.description}",
+                            "type": "string"
+                        }
+                    }
+                },
+                {
+                    "required": [
+                        "modelUrl"
+                    ],
+                    "properties": {
+                        "modelPathType": {
+                            "enum": [
+                                "url"
+                            ]
+                        },
+                        "modelUrl": {
+                            "title": "${strings.activeLearning.form.properties.modelUrl.title}",
+                            "description": "${strings.activeLearning.form.properties.modelUrl.description}",
+                            "default": "http://",
+                            "pattern": "^https?\\\\://[a-zA-Z0-9\\\\-\\\\.]+\\\\.[a-zA-Z]{2,3}(/\\\\S*)?$",
+                            "type": "string"
+                        }
+                    }
+                }
+            ]
+        }
+    }
+}

src/react/components/pages/activeLearning/activeLearningForm.test.tsx

@@ -0,0 +1,95 @@
+import React from "react";
+import { IActiveLearningFormProps, ActiveLearningForm, IActiveLearningFormState } from "./activeLearningForm";
+import { ReactWrapper, mount } from "enzyme";
+import { ModelPathType, IActiveLearningSettings } from "../../../../models/applicationState";
+import Form from "react-jsonschema-form";
+
+describe("Active Learning Form", () => {
+    const onChangeHandler = jest.fn();
+    const onSubmitHandler = jest.fn();
+    const onCancelHandler = jest.fn();
+    const defaultProps: IActiveLearningFormProps = {
+        settings: {
+            modelPathType: ModelPathType.Coco,
+            modelPath: null,
+            modelUrl: null,
+            autoDetect: false,
+            predictTag: true,
+        },
+        onChange: onChangeHandler,
+        onSubmit: onSubmitHandler,
+        onCancel: onCancelHandler,
+    };
+
+    function createComponent(props?: IActiveLearningFormProps)
+        : ReactWrapper<IActiveLearningFormProps, IActiveLearningFormState> {
+        props = props || defaultProps;
+        return mount(<ActiveLearningForm {...props} />);
+    }
+
+    it("renders a dynamic json schema form with default props", () => {
+        const wrapper = createComponent();
+        expect(wrapper.find(Form).exists()).toBe(true);
+        expect(wrapper.state().formData).toEqual(defaultProps.settings);
+    });
+
+    it("sets formData state when loaded with different props", () => {
+        const props: IActiveLearningFormProps = {
+            ...defaultProps,
+            settings: {
+                modelPathType: ModelPathType.Url,
+                modelUrl: "https://myserver.com/myModel",
+                autoDetect: true,
+                predictTag: true,
+            },
+        };
+
+        const wrapper = createComponent(props);
+        expect(wrapper.state().formData).toEqual(props.settings);
+    });
+
+    it("updates form data when the props change", () => {
+        const wrapper = createComponent();
+
+        const newSettings: IActiveLearningSettings = {
+            modelPathType: ModelPathType.Url,
+            modelUrl: "https://myserver.com/myModel",
+            autoDetect: true,
+            predictTag: true,
+        };
+
+        wrapper.setProps({ settings: newSettings });
+        expect(wrapper.state().formData).toEqual(newSettings);
+    });
+
+    it("sets formData state when form changes", () => {
+        const wrapper = createComponent();
+        const formData: IActiveLearningSettings = {
+            modelPathType: ModelPathType.Url,
+            modelUrl: "https://myserver.com/myModel",
+            autoDetect: true,
+            predictTag: true,
+        };
+
+        // Set type to URL
+        wrapper.find(Form).props().onChange({ formData: { modelPathType: ModelPathType.Url } });
+        // Set the remaining settings
+        wrapper.find(Form).props().onChange({ formData });
+        expect(wrapper.state().formData).toEqual(formData);
+        expect(onChangeHandler).toBeCalledWith(formData);
+    });
+
+    it("submits form data to the registered submit handler", () => {
+        const wrapper = createComponent();
+        wrapper.find(Form).props().onSubmit({ formData: defaultProps.settings });
+
+        expect(onSubmitHandler).toBeCalledWith(defaultProps.settings);
+    });
+
+    it("raises the cancel event and called registered handler", () => {
+        const wrapper = createComponent();
+        wrapper.find(".btn-cancel").simulate("click");
+
+        expect(onCancelHandler).toBeCalled();
+    });
+});

src/react/components/pages/activeLearning/activeLearningForm.tsx

@@ -0,0 +1,112 @@
+import React from "react";
+import Form, { ISubmitEvent, IChangeEvent, Widget } from "react-jsonschema-form";
+import { IActiveLearningSettings, ModelPathType } from "../../../../models/applicationState";
+import { strings, addLocValues } from "../../../../common/strings";
+import CustomFieldTemplate from "../../common/customField/customFieldTemplate";
+import LocalFolderPicker from "../../common/localFolderPicker/localFolderPicker";
+import { CustomWidget } from "../../common/customField/customField";
+import Checkbox from "rc-checkbox";
+
+// tslint:disable-next-line:no-var-requires
+const formSchema = addLocValues(require("./activeLearningForm.json"));
+// tslint:disable-next-line:no-var-requires
+const uiSchema = addLocValues(require("./activeLearningForm.ui.json"));
+
+export interface IActiveLearningFormProps extends React.Props<ActiveLearningForm> {
+    settings: IActiveLearningSettings;
+    onSubmit: (settings: IActiveLearningSettings) => void;
+    onChange?: (settings: IActiveLearningSettings) => void;
+    onCancel?: () => void;
+}
+
+export interface IActiveLearningFormState {
+    classNames: string[];
+    formData: IActiveLearningSettings;
+    uiSchema: any;
+    formSchema: any;
+}
+
+export class ActiveLearningForm extends React.Component<IActiveLearningFormProps, IActiveLearningFormState> {
+    public state: IActiveLearningFormState = {
+        classNames: ["needs-validation"],
+        uiSchema: { ...uiSchema },
+        formSchema: { ...formSchema },
+        formData: {
+            ...this.props.settings,
+        },
+    };
+
+    private widgets = {
+        localFolderPicker: (LocalFolderPicker as any) as Widget,
+        checkbox: CustomWidget(Checkbox, (props) => ({
+            checked: props.value,
+            onChange: (value) => props.onChange(value.target.checked),
+            disabled: props.disabled,
+        })),
+    };
+
+    public componentDidUpdate(prevProps: Readonly<IActiveLearningFormProps>) {
+        if (this.props.settings !== prevProps.settings) {
+            this.setState({ formData: this.props.settings });
+        }
+    }
+
+    public render() {
+        return (
+            <Form
+                className={this.state.classNames.join(" ")}
+                showErrorList={false}
+                liveValidate={true}
+                noHtml5Validate={true}
+                FieldTemplate={CustomFieldTemplate}
+                widgets={this.widgets}
+                schema={this.state.formSchema}
+                uiSchema={this.state.uiSchema}
+                formData={this.state.formData}
+                onChange={this.onFormChange}
+                onSubmit={this.onFormSubmit}>
+                <div>
+                    <button className="btn btn-success mr-1" type="submit">{strings.projectSettings.save}</button>
+                    <button className="btn btn-secondary btn-cancel"
+                        type="button"
+                        onClick={this.onFormCancel}>{strings.common.cancel}</button>
+                </div>
+            </Form>
+        );
+    }
+
+    private onFormChange = (changeEvent: IChangeEvent<IActiveLearningSettings>): void => {
+        let updatedSettings = changeEvent.formData;
+
+        if (changeEvent.formData.modelPathType !== this.state.formData.modelPathType) {
+            updatedSettings = {
+                ...changeEvent.formData,
+                modelPath: null,
+                modelUrl: null,
+            };
+        }
+
+        this.setState({
+            formData: updatedSettings,
+        }, () => {
+            if (this.props.onChange) {
+                this.props.onChange(updatedSettings);
+            }
+        });
+    }
+
+    private onFormSubmit = (args: ISubmitEvent<IActiveLearningSettings>): void => {
+        const settings: IActiveLearningSettings = {
+            ...args.formData,
+        };
+
+        this.setState({ formData: settings });
+        this.props.onSubmit(settings);
+    }
+
+    private onFormCancel = (): void => {
+        if (this.props.onCancel) {
+            this.props.onCancel();
+        }
+    }
+}

src/react/components/pages/activeLearning/activeLearningForm.ui.json

@@ -0,0 +1,17 @@
+{
+    "modelPath": {
+        "ui:widget": "localFolderPicker"
+    },
+    "predictTag": {
+        "ui:widget": "checkbox"
+    },
+    "autoDetect": {
+        "ui:widget": "checkbox"
+    },
+    "ui:order": [
+        "modelPathType",
+        "*",
+        "predictTag",
+        "autoDetect"
+    ]
+}

src/react/components/pages/activeLearning/activeLearningPage.test.tsx

@@ -0,0 +1,118 @@
+import React from "react";
+import ActiveLearningPage, { IActiveLearningPageProps, IActiveLearningPageState } from "./activeLearningPage";
+import { ReactWrapper, mount } from "enzyme";
+import { Provider } from "react-redux";
+import { BrowserRouter as Router } from "react-router-dom";
+import createReduxStore from "../../../../redux/store/store";
+import MockFactory from "../../../../common/mockFactory";
+import { ActiveLearningForm } from "./activeLearningForm";
+import { IActiveLearningSettings, ModelPathType } from "../../../../models/applicationState";
+jest.mock("../../../../services/projectService");
+import ProjectService from "../../../../services/projectService";
+import { toast } from "react-toastify";
+import { strings } from "../../../../common/strings";
+
+describe("Active Learning Page", () => {
+    function createComponent(store, props: IActiveLearningPageProps): ReactWrapper {
+        return mount(
+            <Provider store={store}>
+                <Router>
+                    <ActiveLearningPage {...props} />
+                </Router>
+            </Provider>,
+        );
+    }
+
+    beforeAll(() => {
+        toast.success = jest.fn(() => 2);
+    });
+
+    it("renders and loads settings from props", () => {
+        const testProject = MockFactory.createTestProject("TestProject");
+        const store = createReduxStore(MockFactory.initialState({
+            currentProject: testProject,
+        }));
+
+        const props = MockFactory.activeLearningProps();
+        const wrapper = createComponent(store, props);
+
+        const activeLearningPage = wrapper
+            .find(ActiveLearningPage)
+            .childAt(0) as ReactWrapper<IActiveLearningPageProps, IActiveLearningPageState>;
+
+        expect(activeLearningPage.state().settings).toEqual(testProject.activeLearningSettings);
+        expect(wrapper.find(ActiveLearningForm).props().settings).toEqual(testProject.activeLearningSettings);
+    });
+
+    it("updates active learning settings if project changes", () => {
+        const store = createReduxStore(MockFactory.initialState());
+        const props = MockFactory.activeLearningProps();
+
+        const testProject = props.recentProjects[0];
+        const wrapper = createComponent(store, props);
+
+        const activeLearningPage = wrapper
+            .find(ActiveLearningPage)
+            .childAt(0) as ReactWrapper<IActiveLearningPageProps, IActiveLearningPageState>;
+
+        expect(activeLearningPage.state().settings).toEqual(testProject.activeLearningSettings);
+        expect(wrapper.find(ActiveLearningForm).props().settings).toEqual(testProject.activeLearningSettings);
+    });
+
+    it("saves the active learning settings when the form is submitted", async () => {
+        const testProject = MockFactory.createTestProject("TestProject");
+        const activeLearningSettings: IActiveLearningSettings = {
+            ...testProject.activeLearningSettings,
+            modelPathType: ModelPathType.Url,
+            modelUrl: "http://myserver.com/custommodel",
+            autoDetect: true,
+            predictTag: true,
+        };
+
+        const store = createReduxStore(MockFactory.initialState({
+            currentProject: testProject,
+        }));
+
+        const projectServiceMock = ProjectService as jest.Mocked<typeof ProjectService>;
+        projectServiceMock.prototype.load = jest.fn((project) => Promise.resolve(project));
+        projectServiceMock.prototype.save = jest.fn((project) => Promise.resolve(project));
+
+        const props = MockFactory.activeLearningProps();
+        const saveProjectSpy = jest.spyOn(props.actions, "saveProject");
+        saveProjectSpy.mockClear();
+
+        const wrapper = createComponent(store, props);
+
+        const activeLearningForm = wrapper.find(ActiveLearningForm);
+        activeLearningForm.props().onSubmit(activeLearningSettings);
+
+        await MockFactory.flushUi();
+
+        expect(saveProjectSpy).toBeCalledWith(expect.objectContaining({
+            ...testProject,
+            activeLearningSettings,
+        }));
+
+        expect(toast.success).toBeCalledWith(strings.activeLearning.messages.saveSuccess);
+        expect(props.history.goBack).toBeCalled();
+    });
+
+    it("returns to the previous page when the form is cancelled", async () => {
+        const testProject = MockFactory.createTestProject("TestProject");
+        const store = createReduxStore(MockFactory.initialState({
+            currentProject: testProject,
+        }));
+
+        const props = MockFactory.activeLearningProps();
+        const saveProjectSpy = jest.spyOn(props.actions, "saveProject");
+        saveProjectSpy.mockClear();
+
+        const wrapper = createComponent(store, props);
+
+        wrapper.find(ActiveLearningForm).props().onCancel();
+
+        await MockFactory.flushUi();
+        expect(props.history.goBack).toBeCalled();
+        expect(saveProjectSpy).not.toBeCalled();
+    });
+});

src/react/components/pages/activeLearning/activeLearningPage.tsx

@@ -0,0 +1,93 @@
+import React from "react";
+import { connect } from "react-redux";
+import { RouteComponentProps } from "react-router";
+import { bindActionCreators } from "redux";
+import { IActiveLearningSettings, IProject, IApplicationState } from "../../../../models/applicationState";
+import IProjectActions, * as projectActions from "../../../../redux/actions/projectActions";
+import { strings } from "../../../../common/strings";
+import { ActiveLearningForm } from "./activeLearningForm";
+import { toast } from "react-toastify";
+
+export interface IActiveLearningPageProps extends RouteComponentProps, React.Props<ActiveLearningPage> {
+    project: IProject;
+    recentProjects: IProject[];
+    actions: IProjectActions;
+}
+
+export interface IActiveLearningPageState {
+    settings: IActiveLearningSettings;
+}
+
+function mapStateToProps(state: IApplicationState) {
+    return {
+        project: state.currentProject,
+        recentProjects: state.recentProjects,
+    };
+}
+
+function mapDispatchToProps(dispatch) {
+    return {
+        actions: bindActionCreators(projectActions, dispatch),
+    };
+}
+
+@connect(mapStateToProps, mapDispatchToProps)
+export default class ActiveLearningPage extends React.Component<IActiveLearningPageProps, IActiveLearningPageState> {
+    public state: IActiveLearningPageState = {
+        settings: this.props.project ? this.props.project.activeLearningSettings : null,
+    };
+
+    public async componentDidMount() {
+        const projectId = this.props.match.params["projectId"];
+        // If we are creating a new project check to see if there is a partial
+        // project already created in local storage
+        if (!this.props.project && projectId) {
+            const projectToLoad = this.props.recentProjects.find((project) => project.id === projectId);
+            if (projectToLoad) {
+                await this.props.actions.loadProject(projectToLoad);
+            }
+        }
+    }
+
+    public componentDidUpdate(prevProps: Readonly<IActiveLearningPageProps>) {
+        if (prevProps.project !== this.props.project) {
+            this.setState({ settings: this.props.project.activeLearningSettings });
+        }
+    }
+
+    public render() {
+        return (
+            <div className="project-settings-page">
+                <div className="project-settings-page-settings m-3">
+                    <h3>
+                        <i className="fas fa-graduation-cap" />
+                        <span className="px-2">
+                            {strings.activeLearning.title}
+                        </span>
+                    </h3>
+                    <div className="m-3">
+                        <ActiveLearningForm
+                            settings={this.state.settings}
+                            onSubmit={this.onFormSubmit}
+                            onCancel={this.onFormCancel} />
+                    </div>
+                </div>
+            </div>
+        );
+    }
+
+    private onFormSubmit = async (settings: IActiveLearningSettings): Promise<void> => {
+        const updatedProject: IProject = {
+            ...this.props.project,
+            activeLearningSettings: settings,
+        };
+
+        await this.props.actions.saveProject(updatedProject);
+        toast.success(strings.activeLearning.messages.saveSuccess);
+        this.props.history.goBack();
+    }
+
+    private onFormCancel = (): void => {
+        this.props.history.goBack();
+    }
+}

src/react/components/pages/activeLearningPage.tsx

@@ -1,9 +0,0 @@
-import React from "react";
-
-export default class ActiveLearningPage extends React.Component {
-    public render() {
-        return (
-            <div>ActiveLearningPage</div>
-        );
-    }
-}

src/react/components/pages/editorPage/canvas.test.tsx

@@ -45,6 +45,7 @@ describe("Editor Canvas", () => {
         const canvasProps: ICanvasProps = {
             selectedAsset: getAssetMetadata(),
             onAssetMetadataChanged: jest.fn(),
+            onCanvasRendered: jest.fn(),
             editorMode: EditorMode.Rectangle,
             selectionMode: SelectionMode.RECT,
             project: MockFactory.createTestProject(),

src/react/components/pages/editorPage/canvas.tsx

@@ -25,6 +25,7 @@ export interface ICanvasProps extends React.Props<Canvas> {
     children?: ReactElement<AssetPreview>;
     onAssetMetadataChanged?: (assetMetadata: IAssetMetadata) => void;
     onSelectedRegionsChanged?: (regions: IRegion[]) => void;
+    onCanvasRendered?: (canvas: HTMLCanvasElement) => void;
 }
 
 export interface ICanvasState {
@@ -455,6 +456,11 @@ export default class Canvas extends React.Component<ICanvasProps, ICanvasState>
     private setContentSource = async (contentSource: ContentSource) => {
         try {
             await this.editor.addContentSource(contentSource as any);
+
+            if (this.props.onCanvasRendered) {
+                const canvas = this.canvasZone.current.querySelector("canvas");
+                this.props.onCanvasRendered(canvas);
+            }
         } catch (e) {
             console.warn(e);
         }

src/react/components/pages/editorPage/editorPage.test.tsx

@@ -8,7 +8,7 @@ import EditorPage, { IEditorPageProps, IEditorPageState } from "./editorPage";
 import MockFactory from "../../../../common/mockFactory";
 import {
     IApplicationState, IAssetMetadata, IProject,
-    EditorMode, IAsset, AssetState, AssetType, ISize,
+    EditorMode, IAsset, AssetState, ISize, IActiveLearningSettings, ModelPathType,
 } from "../../../../models/applicationState";
 import { AssetProviderFactory } from "../../../../providers/storage/assetProviderFactory";
 import createReduxStore from "../../../../redux/store/store";
@@ -31,6 +31,9 @@ import EditorSideBar from "./editorSideBar";
 import Alert from "../../common/alert/alert";
 import registerMixins from "../../../../registerMixins";
 import { TagInput } from "../../common/tagInput/tagInput";
+import { EditorToolbar } from "./editorToolbar";
+import { ToolbarItem } from "../../toolbar/toolbarItem";
+import { ActiveLearningService } from "../../../../services/activeLearningService";
 
 function createComponent(store, props: IEditorPageProps): ReactWrapper<IEditorPageProps, IEditorPageState, EditorPage> {
     return mount(
@@ -60,9 +63,20 @@ describe("Editor Page Component", () => {
     let assetServiceMock: jest.Mocked<typeof AssetService> = null;
     let projectServiceMock: jest.Mocked<typeof ProjectService> = null;
 
+    const electronMock = {
+        remote: {
+            app: {
+                getAppPath: jest.fn(() => ""),
+            },
+        },
+    };
+
     const testAssets: IAsset[] = MockFactory.createTestAssets(5);
 
     beforeAll(() => {
+        registerToolbar();
+        window["require"] = jest.fn(() => electronMock);
+
         const editorMock = Editor as any;
         editorMock.prototype.addContentSource = jest.fn(() => Promise.resolve());
         editorMock.prototype.scaleRegionToSourceSize = jest.fn((regionData: any) => regionData);
@@ -334,45 +348,6 @@ describe("Editor Page Component", () => {
         expect(saveProjectSpy).toBeCalledWith(expect.objectContaining(partialProject));
     });
 
-    describe("Editor Page Component Forcing Tag Scenario", () => {
-        it("Detect new Tag from asset metadata when selecting the Asset", async () => {
-            const getAssetMetadataMock = assetServiceMock.prototype.getAssetMetadata as jest.Mock;
-            getAssetMetadataMock.mockImplementationOnce((asset) => {
-                const assetMetadata: IAssetMetadata = {
-                    asset: { ...asset },
-                    regions: [{ ...MockFactory.createTestRegion(), tags: ["NEWTAG"] }],
-                    version: appInfo.version,
-                };
-                return Promise.resolve(assetMetadata);
-            });
-
-            // create test project and asset
-            const testProject = MockFactory.createTestProject("TestProject");
-
-            // mock store and props
-            const store = createStore(testProject, true);
-            const props = MockFactory.editorPageProps(testProject.id);
-
-            const saveProjectSpy = jest.spyOn(props.actions, "saveProject");
-
-            // create mock editor page
-            createComponent(store, props);
-
-            const partialProjectToBeSaved = {
-                id: testProject.id,
-                name: testProject.name,
-                tags: expect.arrayContaining([{
-                    name: "NEWTAG",
-                    color: expect.any(String),
-                }]),
-            };
-
-            await MockFactory.flushUi();
-
-            expect(saveProjectSpy).toBeCalledWith(expect.objectContaining(partialProjectToBeSaved));
-        });
-    });
-
     it("When an image is updated the asset metadata is updated", async () => {
         const testProject = MockFactory.createTestProject("TestProject");
         const store = createStore(testProject, true);
@@ -498,7 +473,6 @@ describe("Editor Page Component", () => {
         const removeAllRegionsConfirm = jest.fn();
 
         beforeAll(() => {
-            registerToolbar();
             const clipboard = (navigator as any).clipboard;
             if (!(clipboard && clipboard.writeText)) {
                 (navigator as any).clipboard = {
@@ -826,6 +800,72 @@ describe("Editor Page Component", () => {
             }));
         });
     });
+
+    describe("Active Learning", async () => {
+        let wrapper: ReactWrapper;
+        let editorPage: ReactWrapper<IEditorPageProps, IEditorPageState>;
+        const activeLearningMock = ActiveLearningService as jest.Mocked<typeof ActiveLearningService>;
+
+        async function beforeActiveLearningTest(activeLearningSettings?: IActiveLearningSettings) {
+            document.querySelector = MockFactory.mockCanvas();
+            activeLearningMock.prototype.isModelLoaded = jest.fn(() => true);
+            activeLearningMock.prototype.predictRegions = jest.fn((canvas, assetMetadtata) => {
+                return Promise.resolve({
+                    ...assetMetadtata,
+                    predicted: true,
+                });
+            });
+            const project = MockFactory.createTestProject();
+
+            if (activeLearningSettings) {
+                project.activeLearningSettings = activeLearningSettings;
+            }
+
+            const store = createReduxStore({
+                ...MockFactory.initialState(),
+                currentProject: project,
+            });
+
+            wrapper = createComponent(store, MockFactory.editorPageProps());
+            await waitForSelectedAsset(wrapper);
+            wrapper.update();
+            editorPage = wrapper.find(EditorPage).childAt(0);
+        }
+
+        it("predicts regions when auto detect has been enabled", async () => {
+            const activeLearningSettings: IActiveLearningSettings = {
+                modelPathType: ModelPathType.Coco,
+                autoDetect: true,
+                predictTag: true,
+            };
+
+            await beforeActiveLearningTest(activeLearningSettings);
+
+            editorPage.find(Canvas).props().onCanvasRendered(document.createElement("canvas"));
+            expect(activeLearningMock.prototype.predictRegions).toBeCalled();
+        });
+
+        it("predicts regions when toolbar item is selected", async () => {
+            await beforeActiveLearningTest();
+
+            const toolbarItem = {
+                props: {
+                    name: ToolbarItemName.ActiveLearning,
+                },
+            };
+
+            const selectedAsset = editorPage.state().selectedAsset;
+            wrapper.find(EditorToolbar).props().onToolbarItemSelected(toolbarItem as ToolbarItem);
+
+            await MockFactory.flushUi();
+
+            expect(activeLearningMock.prototype.predictRegions).toBeCalledWith(expect.anything(), selectedAsset);
+            expect(assetServiceMock.prototype.save).toBeCalledWith({
+                ...selectedAsset,
+                predicted: true,
+            });
+        });
+    });
 });
 
 function createStore(project: IProject, setCurrentProject: boolean = false): Store<any, AnyAction> {

src/react/components/pages/editorPage/editorPage.tsx

@@ -10,14 +10,14 @@ import { strings } from "../../../../common/strings";
 import {
     AssetState, AssetType, EditorMode, IApplicationState,
     IAppSettings, IAsset, IAssetMetadata, IProject, IRegion,
-    ISize, ITag,
+    ISize, ITag, IAdditionalPageSettings, AppError, ErrorCode,
 } from "../../../../models/applicationState";
 import { IToolbarItemRegistration, ToolbarItemFactory } from "../../../../providers/toolbar/toolbarItemFactory";
 import IApplicationActions, * as applicationActions from "../../../../redux/actions/applicationActions";
 import IProjectActions, * as projectActions from "../../../../redux/actions/projectActions";
 import { ToolbarItemName } from "../../../../registerToolbar";
 import { AssetService } from "../../../../services/assetService";
-import { AssetPreview, IAssetPreviewSettings } from "../../common/assetPreview/assetPreview";
+import { AssetPreview } from "../../common/assetPreview/assetPreview";
 import { KeyboardBinding } from "../../common/keyboardBinding/keyboardBinding";
 import { KeyEventType } from "../../common/keyboardManager/keyboardManager";
 import { TagInput } from "../../common/tagInput/tagInput";
@@ -29,8 +29,8 @@ import EditorSideBar from "./editorSideBar";
 import { EditorToolbar } from "./editorToolbar";
 import Alert from "../../common/alert/alert";
 import Confirm from "../../common/confirm/confirm";
-// tslint:disable-next-line:no-var-requires
+import { ActiveLearningService } from "../../../../services/activeLearningService";
-const tagColors = require("../../common/tagColors.json");
+import { toast } from "react-toastify";
 
 /**
  * Properties for Editor Page
@@ -64,7 +64,7 @@ export interface IEditorPageState {
     /** The child assets used for nest asset typs */
     childAssets?: IAsset[];
     /** Additional settings for asset previews */
-    additionalSettings?: IAssetPreviewSettings;
+    additionalSettings?: IAdditionalPageSettings;
     /** Most recently selected tag */
     selectedTag: string;
     /** Tags locked for region labeling */
@@ -101,7 +101,6 @@ function mapDispatchToProps(dispatch) {
  */
 @connect(mapStateToProps, mapDispatchToProps)
 export default class EditorPage extends React.Component<IEditorPageProps, IEditorPageState> {
-
     public state: IEditorPageState = {
         selectedTag: null,
         lockedTags: [],
@@ -109,12 +108,16 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
         assets: [],
         childAssets: [],
         editorMode: EditorMode.Rectangle,
-        additionalSettings: { videoSettings: (this.props.project) ? this.props.project.videoSettings : null },
+        additionalSettings: {
+            videoSettings: (this.props.project) ? this.props.project.videoSettings : null,
+            activeLearningSettings: (this.props.project) ? this.props.project.activeLearningSettings : null,
+        },
         thumbnailSize: this.props.appSettings.thumbnailSize || { width: 175, height: 155 },
         isValid: true,
         showInvalidRegionWarning: false,
     };
 
+    private activeLearningService: ActiveLearningService = null;
     private loadingProjectAssets: boolean = false;
     private toolbarItems: IToolbarItemRegistration[] = ToolbarItemFactory.getToolbarItems();
     private canvas: RefObject<Canvas> = React.createRef();
@@ -129,6 +132,8 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
             const project = this.props.recentProjects.find((project) => project.id === projectId);
             await this.props.actions.loadProject(project);
         }
+
+        this.activeLearningService = new ActiveLearningService(this.props.project.activeLearningSettings);
     }
 
     public async componentDidUpdate(prevProps: Readonly<IEditorPageProps>) {
@@ -143,6 +148,7 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
             this.setState({
                 additionalSettings: {
                     videoSettings: (this.props.project) ? this.props.project.videoSettings : null,
+                    activeLearningSettings: (this.props.project) ? this.props.project.activeLearningSettings : null,
                 },
             });
         }
@@ -211,6 +217,7 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
                                         ref={this.canvas}
                                         selectedAsset={this.state.selectedAsset}
                                         onAssetMetadataChanged={this.onAssetMetadataChanged}
+                                        onCanvasRendered={this.onCanvasRendered}
                                         onSelectedRegionsChanged={this.onSelectedRegionsChanged}
                                         editorMode={this.state.editorMode}
                                         selectionMode={this.state.selectionMode}
@@ -479,6 +486,17 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
         this.setState({ childAssets, assets, isValid: true });
     }
 
+    /**
+     * Raised when the asset binary has been painted onto the canvas tools rendering canvas
+     */
+    private onCanvasRendered = async (canvas: HTMLCanvasElement) => {
+        // When active learning auto-detect is enabled
+        // run predictions when asset changes
+        if (this.props.project.activeLearningSettings.autoDetect && !this.state.selectedAsset.asset.predicted) {
+            await this.predictRegions(canvas);
+        }
+    }
+
     private onSelectedRegionsChanged = (selectedRegions: IRegion[]) => {
         this.setState({ selectedRegions });
     }
@@ -540,6 +558,41 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
             case ToolbarItemName.RemoveAllRegions:
                 this.canvas.current.confirmRemoveAllRegions();
                 break;
+            case ToolbarItemName.ActiveLearning:
+                await this.predictRegions();
+                break;
+        }
+    }
+
+    private predictRegions = async (canvas?: HTMLCanvasElement) => {
+        canvas = canvas || document.querySelector("canvas");
+        if (!canvas) {
+            return;
+        }
+
+        // Load the configured ML model
+        if (!this.activeLearningService.isModelLoaded()) {
+            let toastId: number = null;
+            try {
+                toastId = toast.info(strings.activeLearning.messages.loadingModel, { autoClose: false });
+                await this.activeLearningService.ensureModelLoaded();
+            } catch (e) {
+                toast.error(strings.activeLearning.messages.errorLoadModel);
+                return;
+            } finally {
+                toast.dismiss(toastId);
+            }
+        }
+
+        // Predict and add regions to current asset
+        try {
+            const updatedAssetMetadata = await this.activeLearningService
+                .predictRegions(canvas, this.state.selectedAsset);
+
+            await this.onAssetMetadataChanged(updatedAssetMetadata);
+            this.setState({ selectedAsset: updatedAssetMetadata });
+        } catch (e) {
+            throw new AppError(ErrorCode.ActiveLearningPredictionError, "Error predicting regions");
         }
     }
 
@@ -579,7 +632,6 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
         }
 
         const assetMetadata = await this.props.actions.loadAssetMetadata(this.props.project, asset);
-        await this.updateProjectTagsFromAsset(assetMetadata);
 
         try {
             if (!assetMetadata.asset.size) {
@@ -597,32 +649,6 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
         });
     }
 
-    private async updateProjectTagsFromAsset(asset: IAssetMetadata) {
-        const assetTags = new Set();
-        asset.regions.forEach((region) => region.tags.forEach((tag) => assetTags.add(tag)));
-
-        const newTags: ITag[] = this.props.project.tags ? [...this.props.project.tags] : [];
-        let updateTags = false;
-
-        assetTags.forEach((tag) => {
-            if (!this.props.project.tags || this.props.project.tags.length === 0 ||
-                !this.props.project.tags.find((projectTag) => tag === projectTag.name)) {
-                newTags.push({
-                    name: tag,
-                    color: tagColors[newTags.length % tagColors.length],
-                });
-                updateTags = true;
-            }
-        });
-
-        if (updateTags) {
-            asset.asset.state = AssetState.Tagged;
-            const newProject = { ...this.props.project, tags: newTags };
-            await this.props.actions.saveAssetMetadata(newProject, asset);
-            await this.props.actions.saveProject(newProject);
-        }
-    }
-
     private loadProjectAssets = async (): Promise<void> => {
         if (this.loadingProjectAssets || this.state.assets.length > 0) {
             return;

src/react/components/pages/export/exportForm.tsx

@@ -116,7 +116,6 @@ export default class ExportForm extends React.Component<IExportFormProps, IExpor
         if (providerType !== this.state.providerName) {
             this.bindForm(args.formData, true);
         } else {
-            console.log(args.formData);
             this.bindForm(args.formData, false);
         }
     }

src/react/components/pages/profileSettingsPage.tsx

@@ -1,9 +0,0 @@
-import React from "react";
-
-export default class ProfileSettingsPage extends React.Component {
-    public render() {
-        return (
-            <div>ProfileSettingsPage</div>
-        );
-    }
-}

src/react/components/pages/projectSettings/projectForm.tsx

@@ -1,5 +1,5 @@
 import React from "react";
-import Form, { FormValidation, ISubmitEvent, IChangeEvent } from "react-jsonschema-form";
+import Form, { FormValidation, ISubmitEvent, IChangeEvent, Widget } from "react-jsonschema-form";
 import { ITagsInputProps, TagEditorModal, TagsInput } from "vott-react";
 import { addLocValues, strings } from "../../../../common/strings";
 import { IConnection, IProject, ITag, IAppSettings } from "../../../../models/applicationState";
@@ -10,6 +10,7 @@ import CustomFieldTemplate from "../../common/customField/customFieldTemplate";
 import { ISecurityTokenPickerProps, SecurityTokenPicker } from "../../common/securityTokenPicker/securityTokenPicker";
 import "vott-react/dist/css/tagsInput.css";
 import { IConnectionProviderPickerProps } from "../../common/connectionProviderPicker/connectionProviderPicker";
+import LocalFolderPicker from "../../common/localFolderPicker/localFolderPicker";
 
 // tslint:disable-next-line:no-var-requires
 const formSchema = addLocValues(require("./projectForm.json"));
@@ -51,6 +52,10 @@ export interface IProjectFormState {
  * @description - Form for editing or creating VoTT projects
  */
 export default class ProjectForm extends React.Component<IProjectFormProps, IProjectFormState> {
+    private widgets = {
+        localFolderPicker: (LocalFolderPicker as any) as Widget,
+    };
+
     private tagsInput: React.RefObject<TagsInput>;
     private tagEditorModal: React.RefObject<TagEditorModal>;
 
@@ -95,6 +100,7 @@ export default class ProjectForm extends React.Component<IProjectFormProps, IPro
                 FieldTemplate={CustomFieldTemplate}
                 validate={this.onFormValidate}
                 fields={this.fields()}
+                widgets={this.widgets}
                 schema={this.state.formSchema}
                 uiSchema={this.state.uiSchema}
                 formData={this.state.formData}

src/react/components/shell/mainContentRouter.test.tsx

@@ -11,7 +11,6 @@ import MainContentRouter from "./mainContentRouter";
 import HomePage, { IHomePageProps } from "./../pages/homepage/homePage";
 import SettingsPage from "./../pages/appSettings/appSettingsPage";
 import ConnectionsPage from "./../pages/connections/connectionsPage";
-import ProfilePage from "./../pages/profileSettingsPage";
 import { IApplicationState } from "./../../../models/applicationState";
 
 describe("Main Content Router", () => {
@@ -43,7 +42,6 @@ describe("Main Content Router", () => {
         expect(pathMap["/"]).toBe(HomePage);
         expect(pathMap["/settings"]).toBe(SettingsPage);
         expect(pathMap["/connections"]).toBe(ConnectionsPage);
-        expect(pathMap["/profile"]).toBe(ProfilePage);
     });
 
     it("renders a redirect when no route is matched", () => {

src/react/components/shell/mainContentRouter.tsx

@@ -1,13 +1,12 @@
 import React from "react";
-import { Switch, Route, Redirect } from "react-router-dom";
+import { Switch, Route } from "react-router-dom";
 import HomePage from "../pages/homepage/homePage";
-import ActiveLearningPage from "../pages/activeLearningPage";
+import ActiveLearningPage from "../pages/activeLearning/activeLearningPage";
 import AppSettingsPage from "../pages/appSettings/appSettingsPage";
 import ConnectionPage from "../pages/connections/connectionsPage";
 import EditorPage from "../pages/editorPage/editorPage";
 import ExportPage from "../pages/export/exportPage";
 import ProjectSettingsPage from "../pages/projectSettings/projectSettingsPage";
-import ProfileSettingsPage from "../pages/profileSettingsPage";
 
 /**
  * @name - Main Content Router
@@ -19,7 +18,6 @@ export default function MainContentRouter() {
             <Switch>
                 <Route path="/" exact component={HomePage} />
                 <Route path="/settings" component={AppSettingsPage} />
-                <Route path="/profile" component={ProfileSettingsPage} />
                 <Route path="/connections/:connectionId" component={ConnectionPage} />
                 <Route path="/connections" exact component={ConnectionPage} />
                 <Route path="/projects/:projectId/edit" component={EditorPage} />

src/react/components/shell/sidebar.test.tsx

@@ -16,6 +16,6 @@ describe("Sidebar Component", () => {
         expect(wrapper).not.toBeNull();
 
         const links = wrapper.find("ul li");
-        expect(links.length).toEqual(6);
+        expect(links.length).toEqual(7);
     });
 });

src/react/components/shell/sidebar.tsx

@@ -39,7 +39,16 @@ export default function Sidebar({ project }) {
                     <ConditionalNavLink disabled={!projectId}
                         title={strings.export.title}
                         to={`/projects/${projectId}/export`}>
-                        <i className="fas fa-external-link-square-alt"></i></ConditionalNavLink></li>
+                        <i className="fas fa-external-link-square-alt"></i>
+                    </ConditionalNavLink>
+                </li>
+                <li>
+                    <ConditionalNavLink disabled={!projectId}
+                        title={strings.activeLearning.title}
+                        to={`/projects/${projectId}/active-learning`}>
+                        <i className="fas fa-graduation-cap"></i>
+                    </ConditionalNavLink>
+                </li>
                 <li>
                     <NavLink title={strings.connections.title}
                         to={`/connections`}><i className="fas fa-plug"></i></NavLink>

src/redux/actions/projectActions.test.ts

@@ -11,7 +11,7 @@ import ProjectService from "../../services/projectService";
 jest.mock("../../services/assetService");
 import { AssetService } from "../../services/assetService";
 import { ExportProviderFactory } from "../../providers/export/exportProviderFactory";
-import { ExportAssetState, IExportProvider } from "../../providers/export/exportProvider";
+import { IExportProvider } from "../../providers/export/exportProvider";
 import { IApplicationState, IProject } from "../../models/applicationState";
 import initialState from "../store/initialState";
 import { appInfo } from "../../common/appInfo";
@@ -87,40 +87,6 @@ describe("Project Redux Actions", () => {
         expect(result.version).toEqual(appInfo.version);
     });
 
-    it("Save Project action on new project correctly add default export format", async () => {
-        projectServiceMock.prototype.save = jest.fn((project) => Promise.resolve(project));
-
-        const skeletonProject = MockFactory.createTestProject("TestProject");
-        const project = {
-            ...skeletonProject,
-            exportFormat: null,
-        };
-
-        const result = await projectActions.saveProject(project)(store.dispatch, store.getState);
-
-        expect(result.exportFormat).toEqual({
-            providerType: "vottJson",
-            providerOptions: {
-                assetState: ExportAssetState.Visited,
-                includeImages: true,
-            },
-        });
-    });
-
-    it("Save Project action on new project correctly set tags to empty if none created", async () => {
-        projectServiceMock.prototype.save = jest.fn((project) => Promise.resolve(project));
-
-        const skeletonProject = MockFactory.createTestProject("TestProject");
-        const project = {
-            ...skeletonProject,
-            tags: null,
-        };
-
-        const result = await projectActions.saveProject(project)(store.dispatch, store.getState);
-
-        expect(result.tags).toEqual([]);
-    });
-
     it("Save Project action does not override existing export format", async () => {
         projectServiceMock.prototype.save = jest.fn((project) => Promise.resolve(project));
 

src/redux/actions/projectActions.ts

@@ -80,24 +80,7 @@ export function saveProject(project: IProject)
             throw new AppError(ErrorCode.SecurityTokenNotFound, "Security Token Not Found");
         }
 
-        const defaultExportProviderOptions: IVottJsonExportProviderOptions = {
+        const savedProject = await projectService.save(project, projectToken);
-            assetState: ExportAssetState.Visited,
-            includeImages: true,
-        };
-
-        const defaultExportFormat: IExportFormat = {
-            providerType: "vottJson",
-            providerOptions: defaultExportProviderOptions,
-        };
-
-        const newProject = {
-            ...project,
-            version: appInfo.version,
-            exportFormat: project.exportFormat || defaultExportFormat,
-            tags: project.tags || [],
-        };
-
-        const savedProject = await projectService.save(newProject, projectToken);
         dispatch(saveProjectAction(savedProject));
 
         // Reload project after save actions

src/redux/reducers/currentProjectReducer.test.ts

@@ -1,6 +1,6 @@
 import _ from "lodash";
 import { reducer } from "./currentProjectReducer";
-import { IProject, IAssetMetadata, AssetState } from "../../models/applicationState";
+import { IProject, IAssetMetadata, AssetState, ITag } from "../../models/applicationState";
 import MockFactory from "../../common/mockFactory";
 import {
     loadProjectAction,
@@ -50,7 +50,7 @@ describe("Current Project Reducer", () => {
         expect(result).toEqual(currentProject);
     });
 
-    it("Updating connection used by current project is updated in curren project", () => {
+    it("Updating connection used by current project is updated in current project", () => {
         const currentProject = MockFactory.createTestProject("1");
         const state: IProject = currentProject;
 
@@ -113,6 +113,29 @@ describe("Current Project Reducer", () => {
         expect(result.assets[testAssets[0].id]).toEqual(assetMetadata.asset);
     });
 
+    it("Appends new tags to project when saving asset contains new tags", () => {
+        const state: IProject = MockFactory.createTestProject("TestProject");
+        const testAssets = MockFactory.createTestAssets();
+
+        const expectedTag: ITag = {
+            name: "NEWTAG",
+            color: expect.any(String),
+        };
+
+        const assetMetadata = MockFactory.createTestAssetMetadata(
+            testAssets[0],
+            [MockFactory.createTestRegion("Region 1", [expectedTag.name])],
+        );
+
+        const action = saveAssetMetadataAction(assetMetadata);
+        const result = reducer(state, action);
+        expect(result).not.toBe(state);
+        expect(result.tags).toEqual([
+            ...state.tags,
+            expectedTag,
+        ]);
+    });
+
     it("Unknown action performs a noop", () => {
         const state: IProject = MockFactory.createTestProject("TestProject");
         const action = anyOtherAction();

src/redux/reducers/currentProjectReducer.ts

@@ -1,7 +1,9 @@
 import _ from "lodash";
 import { ActionTypes } from "../actions/actionTypes";
-import { IProject } from "../../models/applicationState";
+import { IProject, ITag } from "../../models/applicationState";
 import { AnyAction } from "../actions/actionCreators";
+// tslint:disable-next-line:no-var-requires
+const tagColors = require("../../react/components/common/tagColors.json");
 
 /**
  * Reducer for project. Actions handled:
@@ -38,6 +40,31 @@ export const reducer = (state: IProject = null, action: AnyAction): IProject =>
             const updatedAssets = { ...state.assets } || {};
             updatedAssets[action.payload.asset.id] = { ...action.payload.asset };
 
+            const assetTags = new Set();
+            action.payload.regions.forEach((region) => region.tags.forEach((tag) => assetTags.add(tag)));
+
+            const newTags: ITag[] = state.tags ? [...state.tags] : [];
+            let updateTags = false;
+
+            assetTags.forEach((tag) => {
+                if (!state.tags || state.tags.length === 0 ||
+                    !state.tags.find((projectTag) => tag === projectTag.name)) {
+                    newTags.push({
+                        name: tag,
+                        color: tagColors[newTags.length % tagColors.length],
+                    });
+                    updateTags = true;
+                }
+            });
+
+            if (updateTags) {
+                return {
+                    ...state,
+                    tags: newTags,
+                    assets: updatedAssets,
+                };
+            }
+
             return {
                 ...state,
                 assets: updatedAssets,

src/registerToolbar.ts

@@ -17,6 +17,7 @@ export enum ToolbarItemName {
     NextAsset = "navigateNextAsset",
     SaveProject = "saveProject",
     ExportProject = "exportProject",
+    ActiveLearning = "activeLearning",
 }
 
 export enum ToolbarItemGroup {
@@ -102,6 +103,15 @@ export default function registerToolbar() {
         accelerators: ["CmdOrCtrl+Delete", "CmdOrCtrl+Backspace"],
     });
 
+    ToolbarItemFactory.register({
+        name: ToolbarItemName.ActiveLearning,
+        tooltip: strings.editorPage.toolbar.activeLearning,
+        icon: "fas fa-graduation-cap",
+        group: ToolbarItemGroup.Canvas,
+        type: ToolbarItemType.Action,
+        accelerators: ["CmdOrCtrl+D", "CmdOrCtrl+d"],
+    });
+
     ToolbarItemFactory.register({
         name: ToolbarItemName.PreviousAsset,
         tooltip: strings.editorPage.toolbar.previousAsset,

src/services/activeLearningService.test.ts

@@ -0,0 +1,120 @@
+import { ActiveLearningService } from "./activeLearningService";
+import { IActiveLearningSettings, ModelPathType, IAssetMetadata, AssetState } from "../models/applicationState";
+import MockFactory from "../common/mockFactory";
+import { appInfo } from "../common/appInfo";
+import { ObjectDetection } from "../providers/activeLearning/objectDetection";
+
+describe("Active Learning Service", () => {
+    const objectDetectionMock = ObjectDetection as jest.Mocked<typeof ObjectDetection>;
+    const defaultSettings: IActiveLearningSettings = {
+        modelPathType: ModelPathType.Coco,
+        autoDetect: true,
+        predictTag: true,
+    };
+
+    let activeLearningService: ActiveLearningService = null;
+
+    const electronMock = {
+        remote: {
+            app: {
+                getAppPath: jest.fn(),
+            },
+        },
+    };
+
+    beforeAll(() => {
+        window["require"] = jest.fn(() => electronMock);
+    });
+
+    beforeEach(() => {
+        activeLearningService = new ActiveLearningService(defaultSettings);
+        objectDetectionMock.prototype.load = jest.fn(() => Promise.resolve());
+        objectDetectionMock.prototype.predictImage = jest.fn(() => Promise.resolve([]));
+    });
+
+    it("Predicts new regions to the asset metadata", async () => {
+        objectDetectionMock.prototype.predictImage = jest.fn(() => Promise.resolve(expectedRegions));
+
+        const expectedRegions = MockFactory.createTestRegions(2);
+        const canvas = MockFactory.mockCanvas()();
+        const asset = MockFactory.createTestAsset("TestAsset", AssetState.Visited);
+        const assetMetadata: IAssetMetadata = {
+            asset: {
+                ...asset,
+                state: AssetState.Tagged,
+            },
+            regions: [],
+            version: appInfo.version,
+        };
+
+        const updatedAssetMetadata = await activeLearningService.predictRegions(canvas, assetMetadata);
+
+        expect(updatedAssetMetadata).toEqual({
+            asset: {
+                ...assetMetadata.asset,
+                predicted: true,
+            },
+            regions: expectedRegions,
+            version: appInfo.version,
+        });
+    });
+
+    it("Predicts non matching regions to the asset metadata", async () => {
+        objectDetectionMock.prototype.predictImage = jest.fn(() => Promise.resolve(expectedRegions));
+
+        const uniqueRegion = MockFactory.createTestRegion("UniqueRegion", ["tag1", "tag2"]);
+        const expectedRegions = MockFactory.createTestRegions(4);
+        const canvas = MockFactory.mockCanvas()();
+        const asset = MockFactory.createTestAsset("TestAsset", AssetState.Visited);
+        const assetMetadata: IAssetMetadata = {
+            asset: {
+                ...asset,
+                state: AssetState.Tagged,
+            },
+            regions: [
+                uniqueRegion,
+                expectedRegions[0],
+                expectedRegions[1],
+            ],
+            version: appInfo.version,
+        };
+
+        const updatedAssetMetadata = await activeLearningService.predictRegions(canvas, assetMetadata);
+
+        expect(updatedAssetMetadata).toEqual({
+            asset: {
+                ...assetMetadata.asset,
+                predicted: true,
+            },
+            regions: [
+                uniqueRegion,
+                ...expectedRegions,
+            ],
+            version: appInfo.version,
+        });
+    });
+
+    it("ensures the underlying object detection model is only loaded 1 time", async () => {
+        const canvas = MockFactory.mockCanvas()();
+        const assetMetadata: IAssetMetadata = {
+            asset: MockFactory.createTestAsset("TestAsset", AssetState.Visited),
+            regions: [],
+            version: appInfo.version,
+        };
+
+        await activeLearningService.predictRegions(canvas, assetMetadata);
+        await activeLearningService.predictRegions(canvas, assetMetadata);
+        await activeLearningService.predictRegions(canvas, assetMetadata);
+        await activeLearningService.predictRegions(canvas, assetMetadata);
+        expect(objectDetectionMock.prototype.load).toBeCalledTimes(1);
+    });
+
+    it("fails if constructor requirements aren't satisfied", () => {
+        expect(() => new ActiveLearningService(null)).toThrow();
+    });
+
+    it("fails if method requirements aren't satisfied", () => {
+        const service = new ActiveLearningService(defaultSettings);
+        expect(service.predictRegions(null, null)).rejects.not.toBeNull();
+    });
+});

src/services/activeLearningService.ts

@@ -0,0 +1,104 @@
+import { IAssetMetadata, ModelPathType, IActiveLearningSettings, AssetState } from "../models/applicationState";
+import { ObjectDetection } from "../providers/activeLearning/objectDetection";
+import Guard from "../common/guard";
+import { isElectron } from "../common/hostProcess";
+import { Env } from "../common/environment";
+
+export class ActiveLearningService {
+    private objectDetection: ObjectDetection;
+    private modelLoaded: boolean = false;
+
+    constructor(private settings: IActiveLearningSettings) {
+        Guard.null(settings);
+        this.objectDetection = new ObjectDetection();
+    }
+
+    public isModelLoaded() {
+        return this.modelLoaded;
+    }
+
+    public async predictRegions(canvas: HTMLCanvasElement, assetMetadata: IAssetMetadata): Promise<IAssetMetadata> {
+        Guard.null(canvas);
+        Guard.null(assetMetadata);
+
+        // If the canvas or asset are invalid return asset metadata
+        if (!(canvas.width && canvas.height && assetMetadata.asset && assetMetadata.asset.size)) {
+            return assetMetadata;
+        }
+
+        await this.ensureModelLoaded();
+
+        const xRatio = assetMetadata.asset.size.width / canvas.width;
+        const yRatio = assetMetadata.asset.size.height / canvas.height;
+        const predictedRegions = await this.objectDetection.predictImage(
+            canvas,
+            this.settings.predictTag,
+            xRatio,
+            yRatio,
+        );
+
+        const updatedRegions = [...assetMetadata.regions];
+        predictedRegions.forEach((prediction) => {
+            const matchingRegion = updatedRegions.find((region) => {
+                return region.boundingBox
+                    && region.boundingBox.left === prediction.boundingBox.left
+                    && region.boundingBox.top === prediction.boundingBox.top
+                    && region.boundingBox.width === prediction.boundingBox.width
+                    && region.boundingBox.height === prediction.boundingBox.height;
+            });
+
+            if (updatedRegions.length === 0 || !matchingRegion) {
+                updatedRegions.push(prediction);
+            }
+        });
+
+        return {
+            ...assetMetadata,
+            regions: updatedRegions,
+            asset: {
+                ...assetMetadata.asset,
+                state: updatedRegions.length > 0 ? AssetState.Tagged : AssetState.Visited,
+                predicted: true,
+            },
+        } as IAssetMetadata;
+    }
+
+    public async ensureModelLoaded(): Promise<void> {
+        if (this.modelLoaded) {
+            return Promise.resolve();
+        }
+
+        await this.loadModel();
+        this.modelLoaded = true;
+    }
+
+    private async loadModel() {
+        let modelPath = "";
+        if (this.settings.modelPathType === ModelPathType.Coco) {
+            if (isElectron()) {
+                const appPath = this.getAppPath();
+
+                if (Env.get() !== "production") {
+                    modelPath = appPath + "/cocoSSDModel";
+                } else {
+                    modelPath = appPath + "/../../cocoSSDModel";
+                }
+            } else {
+                modelPath = "https://vott.blob.core.windows.net/coco-ssd-model";
+            }
+        } else if (this.settings.modelPathType === ModelPathType.File) {
+            if (isElectron()) {
+                modelPath = this.settings.modelPath;
+            }
+        } else {
+            modelPath = this.settings.modelUrl;
+        }
+
+        await this.objectDetection.load(modelPath);
+    }
+
+    private getAppPath = () => {
+        const remote = (window as any).require("electron").remote as Electron.Remote;
+        return remote.app.getAppPath();
+    }
+}

src/services/importService.ts

@@ -2,7 +2,7 @@ import shortid from "shortid";
 import {
     IProject, ITag, IConnection, AppError, ErrorCode,
     IAssetMetadata, IRegion, RegionType, AssetState, IFileInfo,
-    IAsset, AssetType,
+    IAsset, AssetType, ModelPathType,
 } from "../models/applicationState";
 import { IV1Project, IV1Region } from "../models/v1Models";
 import packageJson from "../../package.json";
@@ -66,6 +66,7 @@ export default class ImportService implements IImportService {
             videoSettings: {
                 frameExtractionRate: originalProject.framerate ? Number(originalProject.framerate) : 15,
             },
+            activeLearningSettings: null,
             autoSave: true,
         };
     }

src/services/projectService.test.ts

@@ -2,11 +2,16 @@ import _ from "lodash";
 import ProjectService, { IProjectService } from "./projectService";
 import MockFactory from "../common/mockFactory";
 import { StorageProviderFactory } from "../providers/storage/storageProviderFactory";
-import { IProject, IExportFormat, ISecurityToken, AssetState } from "../models/applicationState";
+import {
+    IProject, IExportFormat, ISecurityToken,
+    AssetState, IActiveLearningSettings, ModelPathType,
+} from "../models/applicationState";
 import { constants } from "../common/constants";
 import { ExportProviderFactory } from "../providers/export/exportProviderFactory";
 import { generateKey } from "../common/crypto";
-import { encryptProject } from "../common/utils";
+import { encryptProject, decryptProject } from "../common/utils";
+import { ExportAssetState } from "../providers/export/exportProvider";
+import { IVottJsonExportProviderOptions } from "../providers/export/vottJson";
 
 describe("Project Service", () => {
     let projectSerivce: IProjectService = null;
@@ -76,6 +81,45 @@ describe("Project Service", () => {
             expect.any(String));
     });
 
+    it("sets default export settings when not defined", async () => {
+        testProject.exportFormat = null;
+        const result = await projectSerivce.save(testProject, securityToken);
+
+        const vottJsonExportProviderOptions: IVottJsonExportProviderOptions = {
+            assetState: ExportAssetState.Visited,
+            includeImages: true,
+        };
+
+        const expectedExportFormat: IExportFormat = {
+            providerType: "vottJson",
+            providerOptions: vottJsonExportProviderOptions,
+        };
+
+        const decryptedProject = decryptProject(result, securityToken);
+
+        expect(decryptedProject.exportFormat).toEqual(expectedExportFormat);
+    });
+
+    it("sets default active learning setting when not defined", async () => {
+        testProject.activeLearningSettings = null;
+        const result = await projectSerivce.save(testProject, securityToken);
+
+        const activeLearningSettings: IActiveLearningSettings = {
+            autoDetect: false,
+            predictTag: true,
+            modelPathType: ModelPathType.Coco,
+        };
+
+        expect(result.activeLearningSettings).toEqual(activeLearningSettings);
+    });
+
+    it("initializes tags to empty array if not defined", async () => {
+        testProject.tags = null;
+        const result = await projectSerivce.save(testProject, securityToken);
+
+        expect(result.tags).toEqual([]);
+    });
+
     it("Save calls configured export provider save when defined", async () => {
         testProject.exportFormat = {
             providerType: "azureCustomVision",

src/services/projectService.ts

@@ -1,12 +1,17 @@
 import _ from "lodash";
 import shortid from "shortid";
 import { StorageProviderFactory } from "../providers/storage/storageProviderFactory";
-import { IProject, ISecurityToken, AppError, ErrorCode, AssetState } from "../models/applicationState";
+import {
+    IProject, ISecurityToken, AppError,
+    ErrorCode, ModelPathType, IActiveLearningSettings,
+} from "../models/applicationState";
 import Guard from "../common/guard";
 import { constants } from "../common/constants";
 import { ExportProviderFactory } from "../providers/export/exportProviderFactory";
 import { decryptProject, encryptProject } from "../common/utils";
 import packageJson from "../../package.json";
+import { ExportAssetState } from "../providers/export/exportProvider";
+import { IExportFormat } from "vott-react";
 
 /**
  * Functions required for a project service
@@ -20,6 +25,20 @@ export interface IProjectService {
     isDuplicate(project: IProject, projectList: IProject[]): boolean;
 }
 
+const defaultActiveLearningSettings: IActiveLearningSettings = {
+    autoDetect: false,
+    predictTag: true,
+    modelPathType: ModelPathType.Coco,
+};
+
+const defaultExportOptions: IExportFormat = {
+    providerType: "vottJson",
+    providerOptions: {
+        assetState: ExportAssetState.Visited,
+        includeImages: true,
+    },
+};
+
 /**
  * @name - Project Service
  * @description - Functions for dealing with projects
@@ -35,7 +54,23 @@ export default class ProjectService implements IProjectService {
 
         try {
             const loadedProject = decryptProject(project, securityToken);
-            return Promise.resolve(loadedProject);
+
+            // Ensure tags is always initialized to an array
+            if (!loadedProject.tags) {
+                loadedProject.tags = [];
+            }
+
+            // Initialize active learning settings if they don't exist
+            if (!loadedProject.activeLearningSettings) {
+                loadedProject.activeLearningSettings = defaultActiveLearningSettings;
+            }
+
+            // Initialize export settings if they don't exist
+            if (!loadedProject.exportFormat) {
+                loadedProject.exportFormat = defaultExportOptions;
+            }
+
+            return Promise.resolve({ ...loadedProject });
         } catch (e) {
             const error = new AppError(ErrorCode.ProjectInvalidSecurityToken, "Error decrypting project settings");
             return Promise.reject(error);
@@ -54,6 +89,21 @@ export default class ProjectService implements IProjectService {
             project.id = shortid.generate();
         }
 
+        // Ensure tags is always initialized to an array
+        if (!project.tags) {
+            project.tags = [];
+        }
+
+        // Initialize active learning settings if they don't exist
+        if (!project.activeLearningSettings) {
+            project.activeLearningSettings = defaultActiveLearningSettings;
+        }
+
+        // Initialize export settings if they don't exist
+        if (!project.exportFormat) {
+            project.exportFormat = defaultExportOptions;
+        }
+
         project.version = packageJson.version;
 
         const storageProvider = StorageProviderFactory.createFromConnection(project.targetConnection);