Add progress view (incomplete) + store windows server side (faster)

AngularApp/prototype/angular.json

@@ -20,7 +20,6 @@
             "tsConfig": "tsconfig.app.json",
             "aot": true,
             "assets": [
-              "src/favicon.ico",
               "src/assets"
             ],
             "styles": [

AngularApp/prototype/src/app/api.service.ts

@@ -22,8 +22,8 @@ export class ApiService {
   }
 
 // Split data into windows and normalize
-  async createWindows(values, parameters): Promise<any> {
-    const postData = {values, parameters};
+  async createWindows(parameters): Promise<any> {
+    const postData = {parameters};
     const response = await fetch('http://127.0.0.1:5000/create-windows', {
       method: 'POST',
       headers: {
@@ -36,10 +36,9 @@ export class ApiService {
   }
 
 // Generate LSH-tables by hashing each window
-  async createTables(windows, parameters): Promise<any> {
+  async createTables(parameters): Promise<any> {
     console.log("creating tables");
-    const postData = {windows, parameters};
-    console.log("ahh");
+    const postData = {parameters};
     const response = await fetch('http://127.0.0.1:5000/create-tables', {
       method: 'POST',
       headers: {
@@ -52,20 +51,32 @@ export class ApiService {
   }
 
 // Return similar windows based on an input query
-  async getSimilarWindows(window, tables) {
+  async getSimilarWindows(query, tables) {
     const response = await fetch('http://127.0.0.1:5000/query', {
       method: 'POST',
       headers: {
         'Accept': 'application/json',
         'Content-Type': 'application/json'
       },
-      body: JSON.stringify({window, tables})
+      body: JSON.stringify({query, tables})
     });
     return await response.json();
   }
 
-  async updateTables(windows, labelData, tables, parameters): Promise<any> {
-    const postData = {windows, labelData, tables, parameters};
+  async getAverageWindow(windows): Promise<number[][]> {
+    const response = await fetch('http://127.0.0.1:5000/average', {
+      method: 'POST',
+      headers: {
+        'Accept': 'application/json',
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify({windows})
+    });
+    return await response.json();
+  }
+
+  async updateTables(labelData, tables, parameters): Promise<any> {
+    const postData = {labelData, tables, parameters};
     const response = await fetch('http://127.0.0.1:5000/update', {
       method: 'POST',
       headers: {

AngularApp/prototype/src/app/app.component.html

 <div style="display: flex; justify-content: space-between;">
-  <div style="width: 80%;">
+  <div class="left" style="width: 80%">
     <app-overview-window></app-overview-window>
-  </div>
-  <div style="width: 20%;">
-    <app-query-window></app-query-window>
-  </div>
-</div>
-<mat-tab-group animationDuration="0ms" (selectedTabChange)="changeTab($event)">
+    <mat-tab-group animationDuration="0ms" (selectedTabChange)="changeTab($event)">
       <mat-tab label="Samples">
         <app-labeling-window></app-labeling-window>
       </mat-tab>
@@ -19,4 +14,11 @@
       <mat-tab label="Settings">
         <app-settings></app-settings>
       </mat-tab>
-</mat-tab-group>
+    </mat-tab-group>
+  </div>
+  <div style="width: 20%;">
+    <app-query-window></app-query-window>
+    <app-progress-view></app-progress-view>
+  </div>
+</div>
+

AngularApp/prototype/src/app/app.module.ts

@@ -15,6 +15,7 @@ import {MatTabsModule} from '@angular/material/tabs';
 import { LabelsComponent } from './labels/labels.component';
 import { QueryWindowComponent } from './query-window/query-window.component';
 import {ZingchartAngularModule} from 'zingchart-angular';
+import { ProgressViewComponent } from './progress-view/progress-view.component';
 
 PlotlyModule.plotlyjs = PlotlyJS;
 
@@ -27,6 +28,7 @@ PlotlyModule.plotlyjs = PlotlyJS;
     TableOverviewComponent,
     LabelsComponent,
     QueryWindowComponent,
+    ProgressViewComponent,
   ],
   imports: [
     BrowserModule,

AngularApp/prototype/src/app/cache.service.ts

@@ -9,17 +9,17 @@ export class CacheService {
   public rawIndices: string[];
 
   private _currentTab: number;
-  private _windows: number[][];
   private _query = undefined;
   private _labels = {};
   private _tables;
   private _windowSimilarity;
 
-  public windowSize = 200;
+  public windowSize = 500;
   public nrOfTables = 10;
   public hashSize = 5;
   public stepSize = 1;
 
+  public onNewData: EventEmitter<void> = new EventEmitter<void>();
   public onNewSimilarity: EventEmitter<void> = new EventEmitter<void>();
   public onNewLabels: EventEmitter<void> = new EventEmitter<void>();
   public onNewQuery: EventEmitter<void> = new EventEmitter<void>();
@@ -36,40 +36,43 @@ export class CacheService {
 
   async initialize(): Promise<void> {
     await this.getRawData();
-    await this.getWindows();
+    await this.createWindows();
     await this.createTables();
   }
 
   async reset(): Promise<void> {
     this.windowSimilarity = undefined;
-    await this.getWindows();
+    await this.createWindows();
     await this.createTables();
   }
 
   async getRawData(): Promise<void> {
     const rawData: RawData = await this.api.readFile();
-    console.log(rawData);
     this.rawIndices = rawData.index;
     this.rawValues = rawData.values;
+    this.onNewData.emit();
   }
 
-  async getWindows(): Promise<void> {
-    this.windows = await this.api.createWindows(this.rawValues, this.parameters);
-    console.log(this.windows);
+  async createWindows(): Promise<void> {
+    await this.api.createWindows(this.parameters);
+    this.onNewWindows.emit();
   }
 
   async createTables(): Promise<void> {
-    this.tables = await this.api.createTables(this.windows, this.parameters);
-    console.log(this.tables);
+    this.tables = await this.api.createTables(this.parameters);
   }
 
-  async getSimilarWindows(window): Promise<any> {
-    this.windowSimilarity = await this.api.getSimilarWindows(window, this.tables);
+  async getSimilarWindows(query): Promise<any> {
+    this.windowSimilarity = await this.api.getSimilarWindows(query, this.tables);
     return this.windowSimilarity;
   }
 
+  async getAverageWindow(windows): Promise<number[][]> {
+    return await this.api.getAverageWindow(windows);
+  }
+
   async updateTables(): Promise<void> {
-    this.tables = await this.api.updateTables(this.windows, this.labels, this.tables, this.parameters);
+    this.tables = await this.api.updateTables(this.labels, this.tables, this.parameters);
   }
 
   public set query(v) {
@@ -81,16 +84,12 @@ export class CacheService {
     return this._query;
   }
 
-  public set windows(v) {
-    this._windows = v;
-    this.onNewWindows.emit();
-  }
-
   public get windows(): number[][] {
-    return this._windows;
+    return [];
   }
 
   public set tables(v) {
+    console.log(v);
     this._tables = v;
     this.onNewTables.emit();
   }
@@ -110,6 +109,7 @@ export class CacheService {
 
   public set windowSimilarity(v) {
     this._windowSimilarity = v;
+    console.log(v);
     this.onNewSimilarity.emit();
   }
 
@@ -119,7 +119,6 @@ export class CacheService {
 
   public set currentTab(v) {
     this._currentTab = v;
-    console.log(this.currentTab);
     this.onNewTab.emit();
   }
 

AngularApp/prototype/src/app/labeling-window/labeling-window.component.ts

@@ -41,6 +41,15 @@ export class LabelingWindowComponent implements OnInit {
     this.labels[index] = false;
   }
 
+  shuffleArray(array) {
+    for (var i = array.length - 1; i > 0; i--) {
+      const j = Math.floor(Math.random() * (i + 1));
+      const temp = array[i];
+      array[i] = array[j];
+      array[j] = temp;
+    }
+  }
+
   public getTopKSimilar() {
     this.labels = [];
     let abort = false;
@@ -50,18 +59,19 @@ export class LabelingWindowComponent implements OnInit {
     let topk = [];
     let k = this.k;
     const keys = Object.keys(this.windowSimilarity).map(a => Number(a)).sort((a, b) => b - a);
-    for (const key of keys) {
-      const windows = this.windowSimilarity[key];
+    for (let i = 0; i < this.k; i++) {
+      const windows = this.windowSimilarity[keys[i]];
+      this.shuffleArray(windows);
         for (const index of windows) {
           if (this.service.labels[index] !== undefined) {
             continue;
           }
-        topk.push({index, frequency: 100 * key / this.service.nrOfTables});
+          topk.push({index, frequency: 100 * keys[i] / this.service.nrOfTables});
           k -= 1;
           if (k < 1) {
             abort = true;
-          break;
           }
+          break;
         }
         if (abort) {
           break;

AngularApp/prototype/src/app/labels/labels.component.ts

@@ -19,7 +19,6 @@ export class LabelsComponent implements OnInit {
   createSubplots(): void {
     this.goodLabels = [];
     this.badLabels = [];
-    console.log(this.service.labels);
     for (const key of Object.keys(this.service.labels)) {
       const index = Number(key);
       const plot =

AngularApp/prototype/src/app/overview-window/overview-window.component.html

@@ -2,4 +2,4 @@
 <!--  <plotly-plot *ngIf="showPlot" [data]="data" [layout]="layout" (plotly_click)="clicked($event)"></plotly-plot>-->
 </div>
 
-<zingchart-angular [config]="config" [height]="500" [series]="series" (node_click)="clicked($event)"></zingchart-angular>
+<zingchart-angular [config]="config" [height]="200" [series]="series" (node_click)="clicked($event)"></zingchart-angular>

AngularApp/prototype/src/app/overview-window/overview-window.component.ts

@@ -18,7 +18,8 @@ export class OverviewWindowComponent {
   }
 
   async ngOnInit(): Promise<void> {
-    this.service.onNewWindows.subscribe(() => { this.initializePlot(); });
+    this.service.onNewData.subscribe(() => this.initializePlot());
+    this.service.onNewWindows.subscribe(() => {  });
     this.service.onNewTables.subscribe(() => {
       if (this.service.query) {
         this.updatePlot();
@@ -38,13 +39,7 @@ export class OverviewWindowComponent {
     this.config = {
       type: "mixed",
       preview: {
-        'auto-fit': true,
-        handleTop: {
-          alpha	: 0.0
-        },
-        handleBottom: {
-          alpha	: 0.0
-        }
+        // 'auto-fit': true
       },
       'scale-x': {
         zooming: true,
@@ -104,8 +99,7 @@ export class OverviewWindowComponent {
     const opacity: number[]  = [];
 
     // Similarity
-    const windowSimilarity = await this.service.getSimilarWindows(this.service.windows[this.service.query]);
-    console.log(windowSimilarity);
+    const windowSimilarity = await this.service.getSimilarWindows(this.service.query);
     // for (const frequency in windowSimilarity){
     //   for (const index of windowSimilarity[frequency]) {
     //     colors[index] = this.getColor(Number(frequency) / this.service.nrOfTables);
@@ -116,15 +110,12 @@ export class OverviewWindowComponent {
     let goodLabels = [];
     let badLabels = [];
     for (const index in this.service.labels) {
-      console.log(index);
       if (this.service.labels[index]) {
         goodLabels.push(this.data[index]);
       } else {
         badLabels.push(this.data[index]);
       }
     }
-    console.log(goodLabels);
-    console.log(badLabels);
     this.series = [
       {
         type: 'line',

AngularApp/prototype/src/app/progress-view/progress-view.component.html

+<div *ngIf="similarity" class="window">
+  <div class="plots">
+    <div class="subplot" *ngFor="let subplot of plots">
+      <plotly-plot [data]="subplot.data" [layout]="subplot.layout"></plotly-plot>
+    </div>
+  </div>
+</div>

AngularApp/prototype/src/app/progress-view/progress-view.component.ts

+import { Component, OnInit } from '@angular/core';
+import {CacheService} from '../cache.service';
+
+@Component({
+  selector: 'app-progress-view',
+  templateUrl: './progress-view.component.html',
+  styleUrls: ['./progress-view.component.css']
+})
+export class ProgressViewComponent implements OnInit {
+  public plots;
+
+  constructor(private cache: CacheService) { }
+
+  ngOnInit(): void {
+    this.cache.onNewSimilarity.subscribe(() => { this.initializeInfo() });
+  }
+
+  averagePlot(average) {
+    return {
+      data: [{
+        x: [...Array(average.length).keys()],
+        y: average,
+        type: 'line',
+      }],
+      layout: {
+        hovermode: 'closest',
+        autosize: true,
+        margin: {
+          l: 10,
+          r: 10,
+          t: 10,
+          b: 10,
+          pad: 4
+        },
+        xaxis: {
+          showticklabels: false
+        },
+        yaxis: {
+          showticklabels: false
+        },
+        height: 100,
+        width: screen.width * 0.1,
+      }
+    };
+  }
+
+  async initializeInfo(): Promise<void> {
+    this.plots = [];
+    const keys = Object.keys(this.similarity).map(a => Number(a)).sort((a, b) => b - a);
+    const allWindows = [];
+    for (const key of keys) {
+      allWindows.push(this.similarity[key]);
+    }
+    const averages = await this.cache.getAverageWindow(allWindows);
+    this.plots = averages.map(x => this.averagePlot(x));
+  }
+
+  public get similarity() {
+    return this.cache.windowSimilarity;
+  }
+}

AngularApp/prototype/src/app/table-overview/table-overview.component.css

 .window {
   width: 100%;
+  overflow: scroll;
+}
+
+.plots {
   display: flex;
-  flex-wrap: wrap;
 }
 
 .subplot:hover {

AngularApp/prototype/src/app/table-overview/table-overview.component.html

 <div *ngIf="tables" class="window">
+  <div class="plots">
     <div class="subplot" *ngFor="let subplot of subplots">
       <plotly-plot [data]="subplot.data" [layout]="subplot.layout"></plotly-plot>
     </div>
+  </div>
+  <div class="plots">
+    <div class="subplot" *ngFor="let subplot of averages">
+      <plotly-plot [data]="subplot.data" [layout]="subplot.layout"></plotly-plot>
+    </div>
+  </div>
 </div>

AngularApp/prototype/src/app/table-overview/table-overview.component.ts

 import { Component, OnInit } from '@angular/core';
 import {CacheService} from '../cache.service';
+import {element} from 'protractor';
 
 @Component({
   selector: 'app-table-overview',
@@ -8,20 +9,59 @@ import {CacheService} from '../cache.service';
 })
 export class TableOverviewComponent implements OnInit {
   public subplots;
+  public averages;
   constructor(private service: CacheService) { }
 
   ngOnInit(): void {
     this.service.onNewTables.subscribe(() => this.createPlots());
+    this.service.onNewQuery.subscribe(() => this.createPlots());
   }
 
   public get tables() {
     return this.service.tables;
   }
 
-  createPlots() {
+  averagePlot(average): object {
+    return {
+      data: [{
+        x: [...Array(average.length).keys()],
+        y: average,
+        type: 'line',
+      }],
+      layout: {
+        hovermode: 'closest',
+        autosize: true,
+        margin: {
+          l: 10,
+          r: 10,
+          t: 10,
+          b: 10,
+          pad: 4
+        },
+        xaxis: {
+          showticklabels: false
+        },
+        yaxis: {
+          showticklabels: false
+        },
+        height: 100,
+        width: screen.width * 0.1,
+      }
+    };
+  }
+
+  async createPlots() {
     this.subplots = [];
+    this.averages = [];
+    const listOfWindows = [];
     for (const tableIndex in this.tables) {
       const table = this.tables[tableIndex];
+      if (this.service.query) {
+        const queryEntry = Object.keys(table.entries).filter((hash: string) => {
+          return table.entries[hash].indexOf(this.service.query) > -1;
+        })[0];
+        listOfWindows.push(table.entries[queryEntry]);
+      }
       this.subplots.push(
         {
           data: [{
@@ -29,8 +69,16 @@ export class TableOverviewComponent implements OnInit {
               return Number('0b' + hash);
               }
             ),
-            y: Object.values(table.entries).map((values: number[]) => values.length / this.service.windows.length),
-            type: 'bar'
+            y: Object.values(table.entries).map((values: number[]) => values.length / (this.service.rawValues.length - this.service.windowSize)),
+            type: 'bar',
+            marker: {
+              color: Object.values(table.entries).map((values: number[]) => {
+                if (values.indexOf(this.service.query) > -1) {
+                  return 'red';
+                }
+                return 'blue';
+              })
+            }
           }],
           layout: {
             hovermode: 'closest',
@@ -54,5 +102,10 @@ export class TableOverviewComponent implements OnInit {
         }
       );
     }
+    if (this.service.query) {
+      const temp = await this.service.getAverageWindow(listOfWindows);
+      this.averages = temp.map(x => this.averagePlot(x));
+      console.log(this.averages);
+    }
   }
 }

AngularApp/prototype/src/index.html

@@ -2,10 +2,10 @@
 <html lang="en">
 <head>
   <meta charset="utf-8">
-  <title>Prototype</title>
+  <title>PSEUDo</title>
   <base href="/">
   <meta name="viewport" content="width=device-width, initial-scale=1">
-  <link rel="icon" type="image/x-icon" href="favicon.ico">
+  <link rel="icon" type="image/x-icon" href="assets/favicon.ico">
   <link href="https://fonts.googleapis.com/css?family=Roboto:300,400,500&display=swap" rel="stylesheet">
   <link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
 </head>