Commit 69108089 authored by Kruyff,D.L.W. (Dylan)'s avatar Kruyff,D.L.W. (Dylan)
Browse files

MTS hashing with DTW

parent 068c2d6f
This diff is collapsed.
...@@ -22,6 +22,7 @@ ...@@ -22,6 +22,7 @@
"@angular/platform-browser-dynamic": "~9.1.12", "@angular/platform-browser-dynamic": "~9.1.12",
"@angular/router": "~9.1.12", "@angular/router": "~9.1.12",
"angular-plotly.js": "^2.0.0", "angular-plotly.js": "^2.0.0",
"d3": "^5.0.0",
"plotly.js": "^1.54.6", "plotly.js": "^1.54.6",
"rxjs": "~6.5.4", "rxjs": "~6.5.4",
"tslib": "^1.10.0", "tslib": "^1.10.0",
...@@ -32,9 +33,10 @@ ...@@ -32,9 +33,10 @@
"@angular-devkit/build-angular": "~0.901.11", "@angular-devkit/build-angular": "~0.901.11",
"@angular/cli": "~9.1.11", "@angular/cli": "~9.1.11",
"@angular/compiler-cli": "~9.1.12", "@angular/compiler-cli": "~9.1.12",
"@types/node": "^12.11.1", "@types/d3": "^5.16.3",
"@types/jasmine": "~3.5.0", "@types/jasmine": "~3.5.0",
"@types/jasminewd2": "~2.0.3", "@types/jasminewd2": "~2.0.3",
"@types/node": "^12.11.1",
"codelyzer": "^5.1.2", "codelyzer": "^5.1.2",
"jasmine-core": "~3.5.0", "jasmine-core": "~3.5.0",
"jasmine-spec-reporter": "~4.2.1", "jasmine-spec-reporter": "~4.2.1",
......
...@@ -36,53 +36,51 @@ export class ApiService { ...@@ -36,53 +36,51 @@ export class ApiService {
} }
// Generate LSH-tables by hashing each window // Generate LSH-tables by hashing each window
async createTables(parameters): Promise<any> { async initialize(query): Promise<any> {
console.log("creating tables"); const response = await fetch('http://127.0.0.1:5000/initialize', {
const postData = {parameters};
const response = await fetch('http://127.0.0.1:5000/create-tables', {
method: 'POST', method: 'POST',
headers: { headers: {
'Accept': 'application/json', 'Accept': 'application/json',
'Content-Type': 'application/json' 'Content-Type': 'application/json'
}, },
body: new Blob( [ JSON.stringify(postData) ], { type: 'text/plain' } ) body: new Blob( [ JSON.stringify({query}) ], { type: 'text/plain' } )
}); });
return await response.json(); return await response.json();
} }
async getQueryWindow(window) { // Update
const response = await fetch('http://127.0.0.1:5000/query', { async update(query, hashFunctions, parameters): Promise<any> {
const response = await fetch('http://127.0.0.1:5000/update', {
method: 'POST', method: 'POST',
headers: { headers: {
'Accept': 'application/json', 'Accept': 'application/json',
'Content-Type': 'application/json' 'Content-Type': 'application/json'
}, },
body: JSON.stringify({window}) body: new Blob( [ JSON.stringify({query, hash_functions: hashFunctions, parameters}) ], { type: 'text/plain' } )
}); });
return await response.json(); return await response.json();
} }
async getWindow(indices: number[]) { async getQueryWindow(window) {
const response = await fetch('http://127.0.0.1:5000/window', { const response = await fetch('http://127.0.0.1:5000/query', {
method: 'POST', method: 'POST',
headers: { headers: {
'Accept': 'application/json', 'Accept': 'application/json',
'Content-Type': 'application/json' 'Content-Type': 'application/json'
}, },
body: JSON.stringify({indices}) body: JSON.stringify({window})
}); });
return await response.json(); return await response.json();
} }
// Return similar windows based on an input query async getWindow(indices: number[]) {
async getSimilarWindows(query, tables) { const response = await fetch('http://127.0.0.1:5000/window', {
const response = await fetch('http://127.0.0.1:5000/similarity', {
method: 'POST', method: 'POST',
headers: { headers: {
'Accept': 'application/json', 'Accept': 'application/json',
'Content-Type': 'application/json' 'Content-Type': 'application/json'
}, },
body: JSON.stringify({query, tables}) body: JSON.stringify({indices})
}); });
return await response.json(); return await response.json();
} }
...@@ -110,17 +108,4 @@ export class ApiService { ...@@ -110,17 +108,4 @@ export class ApiService {
}); });
return await response.json(); return await response.json();
} }
async updateTables(query, labelData, tables, parameters): Promise<any> {
const postData = {query, labelData, tables, parameters};
const response = await fetch('http://127.0.0.1:5000/update', {
method: 'POST',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json'
},
body: new Blob( [ JSON.stringify(postData) ], { type: 'text/plain' } )
});
return await response.json();
}
} }
...@@ -16,12 +16,16 @@ export class CacheService { ...@@ -16,12 +16,16 @@ export class CacheService {
private _windowSimilarity; private _windowSimilarity;
private _sliderValue; private _sliderValue;
private _queryWindow; private _queryWindow;
private _candidates: number[];
private _parameters: number[];
public windowSize = 120; public windowSize = 120;
public nrOfTables = 5; public nrOfTables = 5;
public hashSize = 5; public hashSize = 5;
public stepSize = 200; public stepSize = 200;
public querySelectionMode = true; public querySelectionMode = true;
public distances = [];
public dtwdistances: number[] = [];
public onNewData: EventEmitter<void> = new EventEmitter<void>(); public onNewData: EventEmitter<void> = new EventEmitter<void>();
public onNewSimilarity: EventEmitter<void> = new EventEmitter<void>(); public onNewSimilarity: EventEmitter<void> = new EventEmitter<void>();
...@@ -39,19 +43,15 @@ export class CacheService { ...@@ -39,19 +43,15 @@ export class CacheService {
async initialize(): Promise<void> { async initialize(): Promise<void> {
this.loadingProgress = 0; this.loadingProgress = 0;
await this.getRawData(); await this.getRawData();
this.loadingProgress = 30; this.loadingProgress = 50;
await this.createWindows(); await this.createWindows();
this.loadingProgress = 60;
await this.createTables();
this.loadingProgress = 100; this.loadingProgress = 100;
} }
async reset(): Promise<void> { async reset(): Promise<void> {
this.windowSimilarity = undefined; this.windowSimilarity = undefined;
this.loadingProgress = 30; this.loadingProgress = 50;
await this.createWindows(); await this.createWindows();
this.loadingProgress = 60;
await this.createTables();
this.loadingProgress = 100; this.loadingProgress = 100;
} }
...@@ -67,14 +67,22 @@ export class CacheService { ...@@ -67,14 +67,22 @@ export class CacheService {
this.onNewWindows.emit(); this.onNewWindows.emit();
} }
async createTables(): Promise<void> { async initial(): Promise<void> {
this.tables = await this.api.createTables(this.parameters); const output = await this.api.initialize(this._queryWindow);
console.log(this.tables); this.dtwdistances = output.distances;
this.candidates = output.candidates;
this.tables = output.hash_functions;
this._parameters = output.parameters;
console.log(output);
} }
async getSimilarWindows(): Promise<any> { async update(): Promise<void> {
this.windowSimilarity = await this.api.getSimilarWindows(this._queryWindow, this.tables); console.log(this.distances);
return this.windowSimilarity; const output = await this.api.update(this._queryWindow, this.distances, this._parameters);
this.dtwdistances = output.distances;
this.candidates = output.candidates;
this.tables = output.hash_functions;
console.log(output);
} }
async getAverageProgressWindows(windows): Promise<number[][]> { async getAverageProgressWindows(windows): Promise<number[][]> {
...@@ -86,7 +94,9 @@ export class CacheService { ...@@ -86,7 +94,9 @@ export class CacheService {
} }
async getQueryWindow(window): Promise<number[]> { async getQueryWindow(window): Promise<number[]> {
this.queryWindow = await this.api.getQueryWindow(window); const output = await this.api.getQueryWindow(window);
this.distances = output.distances;
this.queryWindow = output.average;
return this._queryWindow; return this._queryWindow;
} }
...@@ -94,10 +104,6 @@ export class CacheService { ...@@ -94,10 +104,6 @@ export class CacheService {
return await this.api.getWindow(indices); return await this.api.getWindow(indices);
} }
async updateTables(): Promise<void> {
this.tables = await this.api.updateTables(this.queryWindow, this.labels, this.tables, this.parameters);
}
public set query(v) { public set query(v) {
this._query = v; this._query = v;
} }
...@@ -164,6 +170,14 @@ export class CacheService { ...@@ -164,6 +170,14 @@ export class CacheService {
return this._queryWindow; return this._queryWindow;
} }
public set candidates(v) {
this._candidates = v;
}
public get candidates() {
return this._candidates;
}
public get parameters(): {[parameter: string]: any} { public get parameters(): {[parameter: string]: any} {
return { return {
windowsize: this.windowSize, windowsize: this.windowSize,
......
<div class="container"> <div class="container">
<div *ngIf="windowSimilarity" class="subplot-container"> <div *ngIf="candidates" class="subplot-container">
<div class="subplot" *ngFor="let subplot of subplots"> <div class="subplot" *ngFor="let subplot of subplots">
<plotly-plot [data]="subplot.data" [layout]="subplot.layout"></plotly-plot> <plotly-plot [data]="subplot.data" [layout]="subplot.layout"></plotly-plot>
<div class="button-holder"> <div class="button-holder">
...@@ -9,7 +9,8 @@ ...@@ -9,7 +9,8 @@
</div> </div>
</div> </div>
</div> </div>
<button *ngIf="windowSimilarity" (click)="train()" class="train-button">Train</button> <button *ngIf="candidates" (click)="updateQuery()" class="train-button">Update query</button>
<button *ngIf="candidates" (click)="train()" class="train-button">Train</button>
</div> </div>
...@@ -15,12 +15,17 @@ export class LabelingWindowComponent implements OnInit { ...@@ -15,12 +15,17 @@ export class LabelingWindowComponent implements OnInit {
constructor(private service: CacheService) { } constructor(private service: CacheService) { }
ngOnInit(): void { ngOnInit(): void {
this.service.onNewSimilarity.subscribe(() => { this.getTopKSimilar(); }); this.service.onNewTables.subscribe(() => { this.getTopKSimilar(); });
} }
public train() { public train() {
this.service.labels = Object.assign({}, this.service.labels, this.labels); this.service.labels = Object.assign({}, this.service.labels, this.labels);
this.service.updateTables(); this.service.update();
}
public updateQuery() {
this.service.labels = Object.assign({}, this.service.labels, this.labels);
this.service.getQueryWindow(this.service.labels);
} }
public labelCorrect(index: number) { public labelCorrect(index: number) {
...@@ -38,58 +43,24 @@ export class LabelingWindowComponent implements OnInit { ...@@ -38,58 +43,24 @@ export class LabelingWindowComponent implements OnInit {
} }
async getTopKSimilar() { async getTopKSimilar() {
this.labels = []; this.topk = this.service.candidates.slice(0, 20);
let abort = false;
const topk = [];
let k = this.k;
const keys = Object.keys(this.windowSimilarity).map(a => Number(a)).sort((a, b) => b - a);
for (let i = 0; i < this.k; i++) {
const windows = this.windowSimilarity[keys[i]];
this.shuffleArray(windows);
for (const index of windows) {
if (this.service.labels[index] !== undefined) {
continue;
}
topk.push({index, frequency: 100 * keys[i] / this.service.nrOfTables});
k -= 1;
if (k < 1) {
abort = true;
break;
}
// break;
}
if (abort) {
break;
}
}
this.topk = topk;
// const candidates = [1254, 483, 103425, 2589, 7524];
const candidates = [ 80503, 8277, 26256, 118148, 22734, 473, 79996, 78224,
33755, 57522];
;
this.topk = [];
for (const candidate of candidates) {
this.topk.push({index: candidate, frequency: 100});
}
console.log(this.topk);
await this.createPlots(); await this.createPlots();
} }
async createPlots() { async createPlots() {
this.subplots = []; this.subplots = [];
const values = await this.service.getWindow(this.topk.map((window) => window.index)); const values = await this.service.getWindow(this.topk);
this.topk.forEach((window, i) => { this.topk.forEach((index, i) => {
this.subplots.push( this.subplots.push(
{ {
index: window.index, index,
data: [{ data: [{
x: [...Array(values[i].length).keys()], x: [...Array(values[i].length).keys()],
y: values[i], y: values[i],
type: 'line' type: 'line'
}], }],
layout: { layout: {
title: `Index: ${window.index.toString()} Similarity: ${window.frequency.toString()}%`, title: `Index: ${index.toString()}`,
hovermode: 'closest', hovermode: 'closest',
autosize: true, autosize: true,
margin: { margin: {
...@@ -119,16 +90,7 @@ export class LabelingWindowComponent implements OnInit { ...@@ -119,16 +90,7 @@ export class LabelingWindowComponent implements OnInit {
}); });
} }
shuffleArray(array) { public get candidates() {
for (var i = array.length - 1; i > 0; i--) { return this.service.candidates;
const j = Math.floor(Math.random() * (i + 1));
const temp = array[i];
array[i] = array[j];
array[j] = temp;
}
}
public get windowSimilarity() {
return this.service.windowSimilarity;
} }
} }
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
<mat-tab-group animationDuration="0ms" (selectedTabChange)="changeTab($event)"> <mat-tab-group animationDuration="0ms" (selectedTabChange)="changeTab($event)">
<mat-tab label="Training"> <mat-tab label="Training">
<app-labeling-window></app-labeling-window> <app-labeling-window></app-labeling-window>
<!-- <app-table-overview></app-table-overview>--> <app-table-overview></app-table-overview>
</mat-tab> </mat-tab>
<mat-tab label="Labeled data"> <mat-tab label="Labeled data">
<app-labels></app-labels> <app-labels></app-labels>
......
...@@ -124,9 +124,6 @@ export class OverviewWindowComponent implements OnInit { ...@@ -124,9 +124,6 @@ export class OverviewWindowComponent implements OnInit {
zingchart.exec("zingchart-ng-1", 'setdata', { zingchart.exec("zingchart-ng-1", 'setdata', {
data: this.config data: this.config
}); });
console.log('querying');
await this.service.getSimilarWindows();
console.log('done');
} }
async updateCandidates(sliderIndex) { async updateCandidates(sliderIndex) {
...@@ -173,11 +170,11 @@ export class OverviewWindowComponent implements OnInit { ...@@ -173,11 +170,11 @@ export class OverviewWindowComponent implements OnInit {
y: clickData.y y: clickData.y
}); });
const index = 80503;// Math.floor(xyInformation[0].scalenumvalue / (12000 / 6)); const index = 80503;// Math.floor(xyInformation[0].scalenumvalue / (12000 / 6));
this.service.queryWindow = await this.service.getQueryWindow(index); await this.service.getQueryWindow(index);
const temp = {}; await this.service.initial();
temp[index] = true; // const temp = {};
this.service.labels = temp; // temp[index] = true;
await this.updatePlot(); // this.service.labels = temp;
} }
zoom(p) { zoom(p) {
......
...@@ -9,3 +9,18 @@ ...@@ -9,3 +9,18 @@
<button class="show-candidates-button" (click)="showCandidates()">Show all {{ amountOfCandidates }} candidates</button> <button class="show-candidates-button" (click)="showCandidates()">Show all {{ amountOfCandidates }} candidates</button>
</div> </div>
</div> </div>
<style>
line { stroke: #999; }
circle { stroke: #fff; stroke-width: 1.5px; }
</style>
<svg width='960' height='500'></svg>
<script src='https://d3js.org/d3.v4.min.js'></script>
<script>
</script>
import { Component, OnInit } from '@angular/core'; import { Component, OnInit } from '@angular/core';
import {CacheService} from '../cache.service'; import {CacheService} from '../cache.service';
import * as d3 from 'd3';
@Component({ @Component({
selector: 'app-progress-view', selector: 'app-progress-view',
...@@ -17,6 +18,7 @@ export class ProgressViewComponent implements OnInit { ...@@ -17,6 +18,7 @@ export class ProgressViewComponent implements OnInit {
constructor(private cache: CacheService) { } constructor(private cache: CacheService) { }
ngOnInit(): void { ngOnInit(): void {
this.showgraph();
this.cache.onNewSimilarity.subscribe(() => { this.initializeInfo(); }); this.cache.onNewSimilarity.subscribe(() => { this.initializeInfo(); });
} }
...@@ -156,4 +158,85 @@ export class ProgressViewComponent implements OnInit { ...@@ -156,4 +158,85 @@ export class ProgressViewComponent implements OnInit {
public showCandidates() { public showCandidates() {
this.cache.sliderValue = this._sliderValue; this.cache.sliderValue = this._sliderValue;
} }
public showgraph() {
console.log('building');
const svg = d3.select('svg');
const width = +svg.attr('width');
const height = +svg.attr('height');
const color = d3.scaleOrdinal(d3.schemeCategory10);
const simulation = d3.forceSimulation()
.force('link', d3.forceLink().id((d: any) => d.id))
.force('charge', d3.forceManyBody().strength(100)) // Gravity force
.force('collide', d3.forceCollide().radius(25).iterations(3)) // Repulsion force
.force('center', d3.forceCenter(width / 2, height / 2));// Position force
d3.json('assets/miserables.json')
.catch((error) => {
throw error;
})
.then((graph) => {
const link = svg.append('g')
.selectAll('line')
.data(graph.links)
.enter().append('line')
.attr('stroke', 'blue')
.attr('fill', 'purple')