Skip to content
GitLab
Commit 77b8e286 authored by Kruyff,D.L.W. (Dylan)'s avatar Kruyff,D.L.W. (Dylan)
Browse files

MTS hashing with DTW 


Former-commit-id: 69108089
parent d9cad38f
Expand all Hide whitespace changes
Inline Side-by-side
AngularApp/prototype/package-lock.json
View file @ 77b8e286
This diff is collapsed.
AngularApp/prototype/package.json
View file @ 77b8e286
......@@ -22,6 +22,7 @@
"@angular/platform-browser-dynamic": "~9.1.12",
"@angular/router": "~9.1.12",
"angular-plotly.js": "^2.0.0",
"d3": "^5.0.0",
"plotly.js": "^1.54.6",
"rxjs": "~6.5.4",
"tslib": "^1.10.0",
......@@ -32,9 +33,10 @@
"@angular-devkit/build-angular": "~0.901.11",
"@angular/cli": "~9.1.11",
"@angular/compiler-cli": "~9.1.12",
"@types/node": "^12.11.1",
"@types/d3": "^5.16.3",
"@types/jasmine": "~3.5.0",
"@types/jasminewd2": "~2.0.3",
"@types/node": "^12.11.1",
"codelyzer": "^5.1.2",
"jasmine-core": "~3.5.0",
"jasmine-spec-reporter": "~4.2.1",
......
AngularApp/prototype/src/app/api.service.ts
View file @ 77b8e286
......@@ -36,53 +36,51 @@ export class ApiService {
}
// Generate LSH-tables by hashing each window
async createTables(parameters): Promise<any> {
console.log("creating tables");
const postData = {parameters};
const response = await fetch('http://127.0.0.1:5000/create-tables', {
async initialize(query): Promise<any> {
const response = await fetch('http://127.0.0.1:5000/initialize', {
method: 'POST',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json'
},
body: new Blob( [ JSON.stringify(postData) ], { type: 'text/plain' } )
body: new Blob( [ JSON.stringify({query}) ], { type: 'text/plain' } )
});
return await response.json();
}
async getQueryWindow(window) {
const response = await fetch('http://127.0.0.1:5000/query', {
// Update
async update(query, hashFunctions, parameters): Promise<any> {
const response = await fetch('http://127.0.0.1:5000/update', {
method: 'POST',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json'
},
body: JSON.stringify({window})
body: new Blob( [ JSON.stringify({query, hash_functions: hashFunctions, parameters}) ], { type: 'text/plain' } )
});
return await response.json();
}
async getWindow(indices: number[]) {
const response = await fetch('http://127.0.0.1:5000/window', {
async getQueryWindow(window) {
const response = await fetch('http://127.0.0.1:5000/query', {
method: 'POST',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json'
},
body: JSON.stringify({indices})
body: JSON.stringify({window})
});
return await response.json();
}
// Return similar windows based on an input query
async getSimilarWindows(query, tables) {
const response = await fetch('http://127.0.0.1:5000/similarity', {
async getWindow(indices: number[]) {
const response = await fetch('http://127.0.0.1:5000/window', {
method: 'POST',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json'
},
body: JSON.stringify({query, tables})
body: JSON.stringify({indices})
});
return await response.json();
}
......@@ -110,17 +108,4 @@ export class ApiService {
});
return await response.json();
}
async updateTables(query, labelData, tables, parameters): Promise<any> {
const postData = {query, labelData, tables, parameters};
const response = await fetch('http://127.0.0.1:5000/update', {
method: 'POST',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json'
},
body: new Blob( [ JSON.stringify(postData) ], { type: 'text/plain' } )
});
return await response.json();
}
}
AngularApp/prototype/src/app/cache.service.ts
View file @ 77b8e286
......@@ -16,12 +16,16 @@ export class CacheService {
private _windowSimilarity;
private _sliderValue;
private _queryWindow;
private _candidates: number[];
private _parameters: number[];
public windowSize = 120;
public nrOfTables = 5;
public hashSize = 5;
public stepSize = 200;
public querySelectionMode = true;
public distances = [];
public dtwdistances: number[] = [];
public onNewData: EventEmitter<void> = new EventEmitter<void>();
public onNewSimilarity: EventEmitter<void> = new EventEmitter<void>();
......@@ -39,19 +43,15 @@ export class CacheService {
async initialize(): Promise<void> {
this.loadingProgress = 0;
await this.getRawData();
this.loadingProgress = 30;
this.loadingProgress = 50;
await this.createWindows();
this.loadingProgress = 60;
await this.createTables();
this.loadingProgress = 100;
}
async reset(): Promise<void> {
this.windowSimilarity = undefined;
this.loadingProgress = 30;
this.loadingProgress = 50;
await this.createWindows();
this.loadingProgress = 60;
await this.createTables();
this.loadingProgress = 100;
}
......@@ -67,14 +67,22 @@ export class CacheService {
this.onNewWindows.emit();
}
async createTables(): Promise<void> {
this.tables = await this.api.createTables(this.parameters);
console.log(this.tables);
async initial(): Promise<void> {
const output = await this.api.initialize(this._queryWindow);
this.dtwdistances = output.distances;
this.candidates = output.candidates;
this.tables = output.hash_functions;
this._parameters = output.parameters;
console.log(output);
}
async getSimilarWindows(): Promise<any> {
this.windowSimilarity = await this.api.getSimilarWindows(this._queryWindow, this.tables);
return this.windowSimilarity;
async update(): Promise<void> {
console.log(this.distances);
const output = await this.api.update(this._queryWindow, this.distances, this._parameters);
this.dtwdistances = output.distances;
this.candidates = output.candidates;
this.tables = output.hash_functions;
console.log(output);
}
async getAverageProgressWindows(windows): Promise<number[][]> {
......@@ -86,7 +94,9 @@ export class CacheService {
}
async getQueryWindow(window): Promise<number[]> {
this.queryWindow = await this.api.getQueryWindow(window);
const output = await this.api.getQueryWindow(window);
this.distances = output.distances;
this.queryWindow = output.average;
return this._queryWindow;
}
......@@ -94,10 +104,6 @@ export class CacheService {
return await this.api.getWindow(indices);
}
async updateTables(): Promise<void> {
this.tables = await this.api.updateTables(this.queryWindow, this.labels, this.tables, this.parameters);
}
public set query(v) {
this._query = v;
}
......@@ -164,6 +170,14 @@ export class CacheService {
return this._queryWindow;
}
public set candidates(v) {
this._candidates = v;
}
public get candidates() {
return this._candidates;
}
public get parameters(): {[parameter: string]: any} {
return {
windowsize: this.windowSize,
......
AngularApp/prototype/src/app/labeling-window/labeling-window.component.html
View file @ 77b8e286
<div class="container">
<div *ngIf="windowSimilarity" class="subplot-container">
<div *ngIf="candidates" class="subplot-container">
<div class="subplot" *ngFor="let subplot of subplots">
<plotly-plot [data]="subplot.data" [layout]="subplot.layout"></plotly-plot>
<div class="button-holder">
......@@ -9,7 +9,8 @@
</div>
</div>
</div>
<button *ngIf="windowSimilarity" (click)="train()" class="train-button">Train</button>
<button *ngIf="candidates" (click)="updateQuery()" class="train-button">Update query</button>
<button *ngIf="candidates" (click)="train()" class="train-button">Train</button>
</div>
AngularApp/prototype/src/app/labeling-window/labeling-window.component.ts
View file @ 77b8e286
......@@ -15,12 +15,17 @@ export class LabelingWindowComponent implements OnInit {
constructor(private service: CacheService) { }
ngOnInit(): void {
this.service.onNewSimilarity.subscribe(() => { this.getTopKSimilar(); });
this.service.onNewTables.subscribe(() => { this.getTopKSimilar(); });
}
public train() {
this.service.labels = Object.assign({}, this.service.labels, this.labels);
this.service.updateTables();
this.service.update();
}
public updateQuery() {
this.service.labels = Object.assign({}, this.service.labels, this.labels);
this.service.getQueryWindow(this.service.labels);
}
public labelCorrect(index: number) {
......@@ -38,58 +43,24 @@ export class LabelingWindowComponent implements OnInit {
}
async getTopKSimilar() {
this.labels = [];
let abort = false;
const topk = [];
let k = this.k;
const keys = Object.keys(this.windowSimilarity).map(a => Number(a)).sort((a, b) => b - a);
for (let i = 0; i < this.k; i++) {
const windows = this.windowSimilarity[keys[i]];
this.shuffleArray(windows);
for (const index of windows) {
if (this.service.labels[index] !== undefined) {
continue;
}
topk.push({index, frequency: 100 * keys[i] / this.service.nrOfTables});
k -= 1;
if (k < 1) {
abort = true;
break;
}
// break;
}
if (abort) {
break;
}
}
this.topk = topk;
// const candidates = [1254, 483, 103425, 2589, 7524];
const candidates = [ 80503, 8277, 26256, 118148, 22734, 473, 79996, 78224,
33755, 57522];
;
this.topk = [];
for (const candidate of candidates) {
this.topk.push({index: candidate, frequency: 100});
}
console.log(this.topk);
this.topk = this.service.candidates.slice(0, 20);
await this.createPlots();
}
async createPlots() {
this.subplots = [];
const values = await this.service.getWindow(this.topk.map((window) => window.index));
this.topk.forEach((window, i) => {
const values = await this.service.getWindow(this.topk);
this.topk.forEach((index, i) => {
this.subplots.push(
{
index: window.index,
index,
data: [{
x: [...Array(values[i].length).keys()],
y: values[i],
type: 'line'
}],
layout: {
title: `Index: ${window.index.toString()} Similarity: ${window.frequency.toString()}%`,
title: `Index: ${index.toString()}`,
hovermode: 'closest',
autosize: true,
margin: {
......@@ -119,16 +90,7 @@ export class LabelingWindowComponent implements OnInit {
});
}
shuffleArray(array) {
for (var i = array.length - 1; i > 0; i--) {
const j = Math.floor(Math.random() * (i + 1));
const temp = array[i];
array[i] = array[j];
array[j] = temp;
}
}
public get windowSimilarity() {
return this.service.windowSimilarity;
public get candidates() {
return this.service.candidates;
}
}
AngularApp/prototype/src/app/main/main.component.html
View file @ 77b8e286
......@@ -4,7 +4,7 @@
<mat-tab-group animationDuration="0ms" (selectedTabChange)="changeTab($event)">
<mat-tab label="Training">
<app-labeling-window></app-labeling-window>
<!-- <app-table-overview></app-table-overview>-->
<app-table-overview></app-table-overview>
</mat-tab>
<mat-tab label="Labeled data">
<app-labels></app-labels>
......
AngularApp/prototype/src/app/overview-window/overview-window.component.ts
View file @ 77b8e286
......@@ -124,9 +124,6 @@ export class OverviewWindowComponent implements OnInit {
zingchart.exec("zingchart-ng-1", 'setdata', {
data: this.config
});
console.log('querying');
await this.service.getSimilarWindows();
console.log('done');
}
async updateCandidates(sliderIndex) {
......@@ -173,11 +170,11 @@ export class OverviewWindowComponent implements OnInit {
y: clickData.y
});
const index = 80503;// Math.floor(xyInformation[0].scalenumvalue / (12000 / 6));
this.service.queryWindow = await this.service.getQueryWindow(index);
const temp = {};
temp[index] = true;
this.service.labels = temp;
await this.updatePlot();
await this.service.getQueryWindow(index);
await this.service.initial();
// const temp = {};
// temp[index] = true;
// this.service.labels = temp;
}
zoom(p) {
......
AngularApp/prototype/src/app/progress-view/progress-view.component.html
View file @ 77b8e286
......@@ -9,3 +9,18 @@
<button class="show-candidates-button" (click)="showCandidates()">Show all {{ amountOfCandidates }} candidates</button>
</div>
</div>
<style>
line { stroke: #999; }
circle { stroke: #fff; stroke-width: 1.5px; }
</style>
<svg width='960' height='500'></svg>
<script src='https://d3js.org/d3.v4.min.js'></script>
<script>
</script>
AngularApp/prototype/src/app/progress-view/progress-view.component.ts
View file @ 77b8e286
import { Component, OnInit } from '@angular/core';
import {CacheService} from '../cache.service';
import * as d3 from 'd3';
@Component({
selector: 'app-progress-view',
......@@ -17,6 +18,7 @@ export class ProgressViewComponent implements OnInit {
constructor(private cache: CacheService) { }
ngOnInit(): void {
this.showgraph();
this.cache.onNewSimilarity.subscribe(() => { this.initializeInfo(); });
}
......@@ -156,4 +158,85 @@ export class ProgressViewComponent implements OnInit {
public showCandidates() {
this.cache.sliderValue = this._sliderValue;
}
public showgraph() {
console.log('building');
const svg = d3.select('svg');
const width = +svg.attr('width');
const height = +svg.attr('height');
const color = d3.scaleOrdinal(d3.schemeCategory10);
const simulation = d3.forceSimulation()
.force('link', d3.forceLink().id((d: any) => d.id))
.force('charge', d3.forceManyBody().strength(100)) // Gravity force
.force('collide', d3.forceCollide().radius(25).iterations(3)) // Repulsion force
.force('center', d3.forceCenter(width / 2, height / 2));// Position force
d3.json('assets/miserables.json')
.catch((error) => {
throw error;
})
.then((graph) => {
const link = svg.append('g')
.selectAll('line')
.data(graph.links)
.enter().append('line')
.attr('stroke', 'blue')
.attr('fill', 'purple')
.attr('stroke-width', (d: any) => d.value * .08);
const node = svg.append('g')
.selectAll('circle')
.data(graph.nodes)
.enter().append('circle')
.attr('r', 5)
.attr('fill', (d: any) => color(d.group))
.call(d3.drag()
.on('start', dragstarted)
.on('drag', dragged)
.on('end', dragended));
simulation
.nodes(graph.nodes)
.on('tick', ticked);
simulation.force<any>('link')
.links(graph.links);
function ticked() {
link
.attr('x1', (d: any) => d.source.x)
.attr('y1', (d: any) => d.source.y)
.attr('x2', (d: any) => d.target.x)
.attr('y2', (d: any) => d.target.y);
node
.attr('cx', (d: any) => d.x)
.attr('cy', (d: any) => d.y);
}
});
function dragstarted(d) {
if (!d3.event.active) {
simulation.alphaTarget(0.1).restart();
}
d.fx = d.x;
d.fy = d.y;
}
function dragged(d) {
d.fx = d3.event.x;
d.fy = d3.event.y;
}
function dragended(d) {
if (!d3.event.active) {
simulation.alphaTarget(0);
}
d.fx = null;
d.fy = null;
}
}
}
AngularApp/prototype/src/app/query-window/query-window.component.ts
View file @ 77b8e286
......@@ -21,13 +21,27 @@ export class QueryWindowComponent implements OnInit {
}
initializePlot(): void {
const data = [{
x: [...Array(this.service.queryWindow.length).keys()],
y: this.service.queryWindow,
type: 'line'
}];
if (this.service.distances.length !== 0) {
const max = this.service.queryWindow.map((num, idx) => {
return num + this.service.distances[idx];
});
const min = this.service.queryWindow.map((num, idx) => {
return num - this.service.distances[idx];
});
data.push({
x: [...Array(this.service.queryWindow.length).keys()],
y: this.service.distances,
type: 'bar'
});
}
this.plot =
{
data: [{
x: [...Array(this.service.queryWindow.length).keys()],
y: this.service.queryWindow,
type: 'line'
}],
data,
layout: {
hovermode: 'closest',
autosize: true,
......
AngularApp/prototype/src/app/settings/settings.component.ts
View file @ 77b8e286
......@@ -29,7 +29,7 @@ export class SettingsComponent implements OnInit {
public newTables() {
this.service.hashSize = Number(this._hashSize);
this.service.nrOfTables = Number(this._nrOfTables);
this.service.createTables();
this.service.initial();
}
public get windowSize() {
......
AngularApp/prototype/src/app/table-overview/table-overview.component.ts
View file @ 77b8e286
......@@ -95,70 +95,56 @@ export class TableOverviewComponent implements OnInit {
async createPlots() {
this.subplots = [];
this.averages = [];
const listOfWindows = [];
for (const tableIndex in this.tables) {
const table = this.tables[tableIndex];
let rank = undefined;
let opacity = new Array(Object.keys(table.entries).length);
opacity.fill(1);
const orderedHashes = Object.keys(table.entries).sort((hash1, hash2) => {
return this.distanceMetric(hash1) - this.distanceMetric(hash2);
});
if (this.service.queryWindow) {
// const queryEntry = this.calculateSignature(table.hash, this.service.queryWindow);
// listOfWindows.push(table.entries[queryEntry]);
// opacity = orderedHashes.map((hash: string) => {
// return hash === queryEntry ? 1.0 : 0.5;
// }
// );
// rank = table.entries[queryEntry].length;
let nrOfBuckets = Math.ceil(Math.max(...this.service.dtwdistances) / 50);
const indices = this.service.dtwdistances.map((x) => x > 500 ? 20 : Math.floor(x / 20));
console.log(indices);
const table = {};
this.service.candidates.forEach((candidate, index) => {
if (table[indices[index]] === undefined)
{
console.log('test');
table[indices[index]] = [];
}
this.subplots.push(
{
// rank: rank,
data: [{
x: orderedHashes.map((hash: string, i) => {
return i; // Number('0b' + hash);