Commit f1c33054 authored by Kruyff,D.L.W. (Dylan)'s avatar Kruyff,D.L.W. (Dylan)
Browse files

Added table prototype views + model learner

parent b9056adb
......@@ -6,9 +6,11 @@ export interface RawData {
}
export interface LshData {
candidates: {[bucket: string]: number[]}[];
candidates: number[][][];
tables: {[bucket: string]: number[]}[];
average_candidates: number[];
average_distances: number[];
samples: number[];
distances: number[][][];
hash_functions: number[][][][];
parameters?: number[];
......@@ -62,6 +64,18 @@ export class ApiService {
return await response.json();
}
async getWeights(query: number[][], labels: {[index: number]: boolean}, weights: number[]): Promise<number[]> {
const response = await fetch('http://127.0.0.1:5000/weights', {
method: 'POST',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json'
},
body: new Blob( [ JSON.stringify({query, labels, weights}) ], { type: 'text/plain' } )
});
return await response.json();
}
// Find candidates using LSH with weights
async lshUpdate(query, weights, parameters): Promise<LshData> {
const response = await fetch('http://127.0.0.1:5000/update', {
......
......@@ -10,7 +10,7 @@ export class LabelingWindowComponent implements OnInit {
public topk: number[];
public subplots = [];
public labels: boolean[] = [];
private k = 12;
private k = 5;
constructor(private state: StateService) { }
......@@ -21,7 +21,7 @@ export class LabelingWindowComponent implements OnInit {
async train() {
this.state.labels = Object.assign({}, this.state.labels, this.labels);
await this.state.getQueryWindow(this.state.labels);
await this.state.update();
await this.state.update(Object.assign({}, this.labels));
}
async updateQuery() {
......@@ -44,9 +44,14 @@ export class LabelingWindowComponent implements OnInit {
}
async showSamples() {
this.topk = this.state.lshData.average_candidates
.filter((candidate) => this.state.labels[candidate] !== true)
.slice(0, this.k);
// const allowedCandidates = this.state.lshData.average_candidates.filter((candidate) => this.state.labels[candidate] !== true);
// this.topk = allowedCandidates.slice(0, this.k);
// for (let i = 0; i < this.k; i++) {
// this.topk.push(allowedCandidates[Math.floor(Math.random() * allowedCandidates.length)]);
// }
this.labels = [];
this.topk = this.state.lshData.samples;
this.subplots = [];
const values: number[][][] = await this.state.getWindow(this.topk);
for (const idx in this.topk) {
......
......@@ -148,7 +148,7 @@ export class OverviewWindowComponent implements OnInit {
};
console.log(this.config);
console.log("showing plot");
// await this.debugClicked();
await this.debugClicked();
}
async updatePlot() {
......@@ -256,7 +256,7 @@ export class OverviewWindowComponent implements OnInit {
}
async debugClicked() {
const index = 1234;
const index = 6713;
await this.state.getQueryWindow(index);
await this.state.lshInitial();
}
......
......@@ -13,6 +13,7 @@ export class StateService {
private _queryWindow: number[][];
private _table: {[bucket: string]: number[]}[];
public _averageTable: {[bucket: string]: number[]};
private _weights: number[];
private _currentTab: number;
private _labels = {};
......@@ -69,18 +70,20 @@ export class StateService {
this.lshData = await this.api.lshInitial(this._queryWindow);
console.log('data loaded');
this._lshParameters = this.lshData.parameters;
this._weights = [1, 1, 1];
this.createTable();
}
async update(): Promise<void> {
this.lshData = await this.api.lshUpdate(this._queryWindow, [], this._lshParameters);
async update(labels): Promise<void> {
this._weights = await this.api.getWeights(this._queryWindow, labels, this._weights);
console.log(this._weights);
this.lshData = await this.api.lshUpdate(this._queryWindow, this._weights, this._lshParameters);
this.createTable();
}
async getTableInfo(): Promise<TableInfoData> {
this.tableInfo = await this.api.getTableInfo(Object.values(this._averageTable));
console.log(this.tableInfo);
return this.tableInfo;
async getTableInfo(table: number[][]): Promise<TableInfoData> {
// console.log(this.tableInfo);
return await this.api.getTableInfo(table);
}
async getQueryWindow(windowIndex: number | {[index: number]: boolean}): Promise<number[][]> {
......@@ -93,9 +96,9 @@ export class StateService {
return await this.api.getWindowByIndices(indices);
}
public createTable() {
async createTable() {
console.log('setting table param');
this.table = this.lshData.candidates;
this.table = this.lshData.tables;
console.log('table param set');
const averageTable = {};
const length = this.lshData.average_distances.length;
......@@ -111,7 +114,7 @@ export class StateService {
});
this._averageTable = averageTable;
console.log('table created');
this.getTableInfo();
this.tableInfo = await this.getTableInfo(Object.values(this._averageTable));
}
public set rawData(v: RawData[]) {
......
......@@ -6,7 +6,7 @@
</div>
<div class="plots">
<div class="subplot" *ngFor="let subplot of averages">
<plotly-plot class="plotly-plot" [data]="subplot.data" [layout]="subplot.layout"></plotly-plot>
<plotly-plot class="plotly-plot" [data]="subplot" [layout]="layout"></plotly-plot>
<!-- <div class="button-holder">-->
<!-- <button class="query-button" (click)="setQuery(subplot.data)">&#x21c4;</button>-->
<!-- </div>-->
......
......@@ -9,65 +9,106 @@ import {StateService} from '../state.service';
export class TableOverviewComponent implements OnInit {
public subplots;
public averages;
public layout;
constructor(private state: StateService) { }
ngOnInit(): void {
this.state.onNewTable.subscribe(() => this.createHistograms());
// this.state.onNewTableInfo.subscribe(() => this.createPrototypes());
this.state.onNewTable.subscribe(() => {
this.createHistograms();
this.createPrototypes();
});
}
createPrototypes(): void {
this.averages = this.state.tableInfo.prototypes.map(prototype => {
return {
data: [
{
x: [...Array(prototype.average.length).keys()],
y: prototype.average,
type: 'line',
},
{
x: [...Array(prototype.average.length).keys()],
y: prototype.max,
type: 'scatter',
fill: null,
mode: 'lines',
line: {
color: 'rgb(55, 128, 191)',
width: 3
}
},
{
x: [...Array(prototype.average.length).keys()],
y: prototype.min,
type: 'scatter',
fill: 'tonexty',
mode: 'lines',
line: {
color: 'rgb(55, 128, 191)',
width: 3
}
async createPrototypes(): Promise<void> {
const representatives: number[][] = [];
this.state.lshData.candidates.forEach((grouphash) => {
grouphash.forEach((candidates) => {
representatives.push(candidates.slice(0, 20));
});
});
const prototypes = await this.state.getTableInfo(representatives);
const subplots = [];
this.averages = prototypes.prototypes.map((prototype) => {
const channelData = [];
prototype.max.forEach((channel, index) => {
channelData.push({
x: [...Array(channel.length).keys()],
y: channel,
xaxis: 'x',
yaxis: `y${index + 2}`,
type: 'scatter',
fill: null,
mode: 'lines',
line: {
color: 'rgb(55, 128, 191)',
width: 3
}
],
layout: {
showlegend: false,
hovermode: 'closest',
autosize: true,
margin: {
l: 10,
r: 10,
t: 10,
b: 10,
pad: 4
},
xaxis: {
showticklabels: false
},
yaxis: {
showticklabels: false
},
height: 100,
width: screen.width * 0.1,
}
});
});
prototype.min.forEach((channel, index) => {
channelData.push({
x: [...Array(channel.length).keys()],
y: channel,
xaxis: 'x',
yaxis: `y${index + 2}`,
type: 'scatter',
fill: 'tonexty',
mode: 'lines',
line: {
color: 'rgb(55, 128, 191)',
width: 3
}
});
});
prototype.average.forEach((channel, index) => {
channelData.push({
x: [...Array(channel.length).keys()],
y: channel,
xaxis: 'x',
yaxis: `y${index + 2}`,
type: 'line',
line: {
color: 'red',
width: 3
}
});
});
return channelData;
});
for (let index = 0; index < this.state.queryWindow.length; index++) {
subplots.push([`xy${index + 2}`]);
}
this.layout = {
grid: {
rows: this.state.queryWindow.length,
columns: 1,
subplots: subplots,
},
showlegend: false,
hovermode: 'closest',
autosize: true,
margin: {
l: 10,
r: 10,
t: 30,
pad: 4
},
xaxis: {
showgrid: false,
zeroline: false,
showticklabels: false,
},
yaxis: {
zeroline: false,
showticklabels: false,
},
height: 300,
width: screen.width * 0.1,
};
this.state.queryWindow.forEach((channel: number[], index: number) => {
this.layout[`yaxis${index + 2}`] = {
zeroline: false,
showticklabels: false,
};
});
}
......
......@@ -23,12 +23,16 @@
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/api.service.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/api.service.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/labeling-window/labeling-window.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/labeling-window/labeling-window.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/progress-view/progress-view.component.css" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/progress-view/progress-view.component.css" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/progress-view/progress-view.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/progress-view/progress-view.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/state.service.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/state.service.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/table-overview/table-overview.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/table-overview/table-overview.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/_lsh.cpython-38-x86_64-linux-gnu.so" beforeDir="false" afterPath="$PROJECT_DIR$/_lsh.cpython-38-x86_64-linux-gnu.so" afterDir="false" />
<change beforePath="$PROJECT_DIR$/build/lib.linux-x86_64-3.8/_lsh.cpython-38-x86_64-linux-gnu.so" beforeDir="false" afterPath="$PROJECT_DIR$/build/lib.linux-x86_64-3.8/_lsh.cpython-38-x86_64-linux-gnu.so" afterDir="false" />
<change beforePath="$PROJECT_DIR$/build/temp.linux-x86_64-3.8/locality-sensitive-hashing-visual-analytics/lsh-fast/_lsh.o" beforeDir="false" afterPath="$PROJECT_DIR$/build/temp.linux-x86_64-3.8/locality-sensitive-hashing-visual-analytics/lsh-fast/_lsh.o" afterDir="false" />
<change beforePath="$PROJECT_DIR$/build/temp.linux-x86_64-3.8/locality-sensitive-hashing-visual-analytics/lsh-fast/lsh.o" beforeDir="false" afterPath="$PROJECT_DIR$/build/temp.linux-x86_64-3.8/locality-sensitive-hashing-visual-analytics/lsh-fast/lsh.o" afterDir="false" />
<change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/processed-data.npy" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/../experiments/.ipynb_checkpoints/EEG data test-checkpoint.ipynb" beforeDir="false" afterPath="$PROJECT_DIR$/../experiments/.ipynb_checkpoints/EEG data test-checkpoint.ipynb" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../experiments/EEG data test.ipynb" beforeDir="false" afterPath="$PROJECT_DIR$/../experiments/EEG data test.ipynb" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
......@@ -48,7 +52,7 @@
<property name="RunOnceActivity.OpenProjectViewOnStart" value="true" />
<property name="SHARE_PROJECT_CONFIGURATION_FILES" value="true" />
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="last_opened_file_path" value="$PROJECT_DIR$/../../hashing-based-network-discovery-master" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" />
</component>
......@@ -143,10 +147,10 @@
<screen x="72" y="27" width="1848" height="1053" />
</state>
<state x="479" y="254" width="1200" height="800" key="DiffContextDialog/72.27.1848.1053@72.27.1848.1053" timestamp="1603129938934" />
<state x="779" y="311" width="424" height="491" key="FileChooserDialogImpl" timestamp="1605528689693">
<state x="779" y="311" width="424" height="491" key="FileChooserDialogImpl" timestamp="1606260652750">
<screen x="72" y="27" width="1848" height="1053" />
</state>
<state x="779" y="311" width="424" height="491" key="FileChooserDialogImpl/72.27.1848.1053@72.27.1848.1053" timestamp="1605528689693" />
<state x="779" y="311" width="424" height="491" key="FileChooserDialogImpl/72.27.1848.1053@72.27.1848.1053" timestamp="1606260652750" />
<state x="659" y="259" width="672" height="678" key="search.everywhere.popup" timestamp="1604929652702">
<screen x="72" y="27" width="1848" height="1053" />
</state>
......
......@@ -175,15 +175,19 @@ def initialize():
t0 = time()
raw_data = orjson.loads(request.data)
data = np.load('processed-data.npy')
data = np.reshape(data, (len(data), len(data[0][0]), len(data[0])))
data = np.swapaxes(data, 1, 2)
# data = np.reshape(data, (len(data), len(data[0][0]), len(data[0])))
query = raw_data["query"]
query = np.reshape(query, (len(query[0]), len(query)))
parameters = np.load('parameters.npy')
query = np.swapaxes(query, 0, 1)
# query = np.reshape(query, (len(query[0]), len(query)))
parameters = preprocess(data)
# parameters = np.load('parameters.npy')
r = parameters[0]
a = parameters[1]
sd = parameters[2]
candidates, distances, hf = _lsh.lsh(data, query, r, a, sd)
print(distances)
dict = defaultdict(int)
for l in range(len(candidates)):
......@@ -195,9 +199,11 @@ def initialize():
average_distances = list(sorted_dict.values())
tables = []
samples_set = set()
candidates = candidates.tolist()
for l in range(len(candidates)):
for k in range(len(candidates[0])):
samples_set.update(candidates[l][k][0:5])
dict = defaultdict(list)
length = len(distances[l][k])
median = distances[l][k][math.ceil(length/2)]
......@@ -207,13 +213,17 @@ def initialize():
dict[str(indices[i])].append(candidates[l][k][i])
tables.append(dict)
samples = np.array(list(filter(lambda x: x in samples_set, average_candidates))).tolist()
response = {
"hash_functions": hf.tolist(),
"candidates": tables,
"candidates": candidates,
"tables": tables,
"distances": distances.tolist(),
"samples": list(samples),
"average_candidates": np.array(average_candidates).tolist(),
"average_distances": np.array(average_distances).tolist(),
"distances": distances.tolist(),
"parameters": [float(r), float(a), float(sd)]
}
response = orjson.dumps(response)
......@@ -222,12 +232,40 @@ def initialize():
@app.route('/weights', methods=['POST'])
def weights():
alpha = 0.2
raw_data = orjson.loads(request.data)
parameters = raw_data["labels"]
labels = raw_data["labels"]
query = raw_data["query"]
old_weights = raw_data["weights"]
data = np.load('processed-data.npy')
all_good_windows = data[[[int(index) for index, value in labels.items() if value is True]]]
all_bad_windows = data[[[int(index) for index, value in labels.items() if value is False]]]
good_distances = np.zeros(len(query))
for window in all_good_windows:
for i in range(len(all_good_windows[0])):
good_distances[i] += _ucrdtw.ucrdtw(query[i], window[i], 0.05, False)[1]
if len(all_good_windows) != 0:
good_distances /= np.sum(good_distances)
good_distances = np.ones(len(query)) - good_distances
good_distances /= np.sum(good_distances)
good_distances *= len(all_good_windows[0])
good_distances = np.sqrt(good_distances)
good_distances = alpha * np.array(old_weights) + (1-alpha) * good_distances
# bad_distances = np.zeros(len(query))
# for window in all_bad_windows:
# for i in range(len(all_bad_windows[0])):
# bad_distances[i] += _ucrdtw.ucrdtw(query[i], window[i], 0.05, False)[1]
# if len(all_bad_windows) != 0:
# bad_distances /= np.sum(bad_distances)
# bad_distances = np.ones(len(query)) - bad_distances
print(good_distances)
# Caculate weights
response = weights
response = orjson.dumps(good_distances.tolist())
return response
......@@ -236,13 +274,15 @@ def update():
t0 = time()
raw_data = orjson.loads(request.data)
data = np.load('processed-data.npy')
data = np.reshape(data, (len(data), len(data[0][0]), len(data[0])))
data = np.swapaxes(data, 1, 2)
# data = np.reshape(data, (len(data), len(data[0][0]), len(data[0])))
query = raw_data["query"]
query = np.reshape(query, (len(query[0]), len(query)))
query = np.swapaxes(query, 0, 1)
# query = np.reshape(query, (len(query[0]), len(query)))
weights = raw_data["weights"]
parameters = raw_data["parameters"]
candidates, distances, hf = _lsh.lsh(data, query, parameters[0], parameters[1], parameters[2])
candidates, distances, hf = _lsh.lsh(data, query, parameters[0], parameters[1], parameters[2], weights)
dict = defaultdict(int)
for l in range(len(candidates)):
for k in range(len(candidates[0])):
......@@ -253,9 +293,11 @@ def update():
average_distances = list(sorted_dict.values())
tables = []
samples_set = set()
candidates = candidates.tolist()
for l in range(len(candidates)):
for k in range(len(candidates[0])):
samples_set.update(candidates[l][k][0:5])
dict = defaultdict(list)
length = len(distances[l][k])
median = distances[l][k][math.ceil(length/2)]
......@@ -265,9 +307,13 @@ def update():
dict[str(indices[i])].append(candidates[l][k][i])
tables.append(dict)
samples = np.array(list(filter(lambda x: x in samples_set, average_candidates))).tolist()
response = {
"hash_functions": hf.tolist(),
"candidates": tables,
"candidates": candidates,
"tables": tables,
"samples": list(samples),
"average_candidates": np.array(average_candidates).tolist(),
"average_distances": np.array(average_distances).tolist(),
"distances": distances.tolist(),
......@@ -322,8 +368,8 @@ def table_info():
'max': max_values.tolist(),
'min': min_values.tolist()
})
distances = [[dtw(np.array(v["average"]), np.array(w["average"]), global_constraint='sakoe_chiba', sakoe_chiba_radius=int(0.05 * 120)) for j, w in enumerate(prototypes)] for i, v in enumerate(prototypes)]
response = orjson.dumps({'prototypes': prototypes, 'distances': distances})
# distances = [[dtw(np.array(v["average"]), np.array(w["average"]), global_constraint='sakoe_chiba', sakoe_chiba_radius=int(0.05 * 120)) for j, w in enumerate(prototypes)] for i, v in enumerate(prototypes)]
response = orjson.dumps({'prototypes': prototypes, 'distances': []})
print("Averages calculated: " + str(time() - t0))
return response
......
......@@ -2,9 +2,18 @@
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 26,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The autoreload extension is already loaded. To reload it, use:\n",
" %reload_ext autoreload\n"
]
}
],
"source": [
"%load_ext autoreload\n",
"%autoreload 2"
......@@ -12,7 +21,7 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 1,
"metadata": {},
"outputs": [
{
......@@ -39,45 +48,30 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(59999, 74, 120)\n"
"(59999, 120, 40)\n"
]
}
],
"source": [
"window_data = [npdata[i:i+120] for i in range(0, npdata.shape[0]-120, int(120/8))]\n",
"window_data = [npdata[i:i+120, 0:40] for i in range(0, npdata.shape[0]-120, int(120/8))]\n",
"del npdata\n",
"data = np.reshape(window_data, (len(window_data), len(window_data[0][0]), len(window_data[0])))\n",
"del window_data\n",
"data = np.reshape(data, (len(data), len(data[0][0]), len(data[0])))\n",
"# data = np.concatenate((data, data))\n",
"print(data.shape)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",