Skip to content

GitLab

Commit 8ca0800b authored by Kruyff,D.L.W. (Dylan)'s avatar Kruyff,D.L.W. (Dylan)
Browse files

Fast implementation for debug/testing 


Former-commit-id: b9056adb
parent 4b8763ac
Hide whitespace changes
Inline Side-by-side
AngularApp/prototype/src/app/api.service.ts
View file @ 8ca0800b
......@@ -6,7 +6,7 @@ export interface RawData {
}
export interface LshData {
candidates: number[][][];
candidates: {[bucket: string]: number[]}[];
average_candidates: number[];
average_distances: number[];
distances: number[][][];
......
AngularApp/prototype/src/app/labeling-window/labeling-window.component.ts
View file @ 8ca0800b
......@@ -15,7 +15,7 @@ export class LabelingWindowComponent implements OnInit {
constructor(private state: StateService) { }
ngOnInit(): void {
this.state.onNewTable.subscribe(() => this.showSamples());
this.state.onNewLshData.subscribe(() => this.showSamples());
}
async train() {
......
AngularApp/prototype/src/app/overview-window/overview-window.component.ts
View file @ 8ca0800b
......@@ -26,7 +26,7 @@ export class OverviewWindowComponent implements OnInit {
async ngOnInit(): Promise<void> {
this.state.onNewData.subscribe(() => this.initializePlot());
this.state.onNewTable.subscribe(() => this.updatePlot());
// this.state.onNewTable.subscribe(() => this.updatePlot());
}
async initializePlot() {
......@@ -148,7 +148,7 @@ export class OverviewWindowComponent implements OnInit {
};
console.log(this.config);
console.log("showing plot");
await this.debugClicked();
// await this.debugClicked();
}
async updatePlot() {
......@@ -178,15 +178,15 @@ export class OverviewWindowComponent implements OnInit {
});
}
}
// for (const index of this.state.lshData.average_candidates) {
// this.candidateLabels.push([Number(index), 1]);
// this.markers.push({
// type: 'area',
// // BUG: For some reason the range values are multiplied by 10
// range: [Number(index) / 10, (Number(index) + this.state.windowSize) / 10],
// backgroundColor: '#b1a343',
// });
// }
for (const index of this.state.lshData.average_candidates.slice(0, 100)) {
this.candidateLabels.push([Number(index), 0]);
this.markers.push({
type: 'area',
// BUG: For some reason the range values are multiplied by 10
range: [Number(index) / 10, (Number(index) + this.state.windowSize) / 10],
backgroundColor: '#b1a343',
});
}
console.log(this.markers);
for (const channel of this.config.graphset) {
if (channel.type === 'line') {
......
AngularApp/prototype/src/app/progress-view/progress-view.component.css
View file @ 8ca0800b
......@@ -27,9 +27,12 @@
justify-content: center;
}
mat-slider {
.slider {
display: flex;
justify-content: center;
}
mat-slider {
width: 400px;
}
......
AngularApp/prototype/src/app/progress-view/progress-view.component.ts
View file @ 8ca0800b
......@@ -36,7 +36,11 @@ export class ProgressViewComponent implements OnInit {
marker: {
color: Object.keys(table).map((key) => {
return this.getColor(Number(key) / Number(Object.keys(table)[Object.keys(table).length - 1]));
})
}),
line: {
color: 'black',
width: 0,
}
}
}],
layout: {
......@@ -142,7 +146,7 @@ export class ProgressViewComponent implements OnInit {
zeroline: false,
showticklabels: false,
},
height: 400,
height: 350,
width: 400,
};
this.state.queryWindow.forEach((channel: number[], index: number) => {
......@@ -157,6 +161,11 @@ export class ProgressViewComponent implements OnInit {
this._sliderValue = v.value;
d3.selectAll('circle').transition().style('stroke', undefined);
d3.select('#node-' + v.value).transition().style('stroke', 'black').style('stroke-width', 20);
const data = this.hist;
data.data[0].marker.line.width = Object.keys(this.state._averageTable).map((key) => {
return Number(key) === v.value ? 4 : 0;
});
this.hist = data;
}
public get sliderValue(): number {
......
AngularApp/prototype/src/app/state.service.ts
View file @ 8ca0800b
......@@ -32,6 +32,7 @@ export class StateService {
public onNewQuery: EventEmitter<void> = new EventEmitter<void>();
public onNewTable: EventEmitter<void> = new EventEmitter<void>();
public onNewTableInfo: EventEmitter<void> = new EventEmitter<void>();
public onNewLshData: EventEmitter<void> = new EventEmitter<void>();
public onNewLabels: EventEmitter<void> = new EventEmitter<void>();
public onNewTab: EventEmitter<void> = new EventEmitter<void>();
......@@ -66,6 +67,7 @@ export class StateService {
async lshInitial(): Promise<void> {
this.lshData = await this.api.lshInitial(this._queryWindow);
console.log('data loaded');
this._lshParameters = this.lshData.parameters;
this.createTable();
}
......@@ -83,6 +85,7 @@ export class StateService {
async getQueryWindow(windowIndex: number | {[index: number]: boolean}): Promise<number[][]> {
this.queryWindow = await this.api.getQueryWindow(windowIndex);
console.log(this.queryWindow);
return this._queryWindow;
}
......@@ -91,25 +94,9 @@ export class StateService {
}
public createTable() {
const table = [];
this.lshData.candidates.forEach((groupHash: number[][], i: number) => {
groupHash.forEach((candidates: number[], j: number) => {
const subTable = {};
const length = this.lshData.distances[i][j].length;
const median = this.lshData.distances[i][j][Math.ceil(length / 2)];
const stepsize = median / 10;
const indices: number[] = this.lshData.distances[i][j].map((x) => x > median * 2 ? 19 : Math.floor(x / stepsize));
candidates.forEach((candidate: number, index: number) => {
if (subTable[indices[index]] === undefined)
{
subTable[indices[index]] = [];
}
subTable[indices[index]].push(candidate);
});
table.push(subTable);
});
});
this.table = table;
console.log('setting table param');
this.table = this.lshData.candidates;
console.log('table param set');
const averageTable = {};
const length = this.lshData.average_distances.length;
const median = this.lshData.average_distances[Math.ceil(length / 2)];
......@@ -123,6 +110,7 @@ export class StateService {
averageTable[indices[index]].push(candidate);
});
this._averageTable = averageTable;
console.log('table created');
this.getTableInfo();
}
......@@ -139,6 +127,7 @@ export class StateService {
public set lshData(v: LshData) {
console.log(v);
this._lshData = v;
this.onNewLshData.emit();
}
public get lshData(): LshData {
......@@ -157,6 +146,7 @@ export class StateService {
public set table(v: {[bucket: string]: number[]}[]) {
console.log(v);
this._table = v;
console.log('emitting onNewTable');
this.onNewTable.emit();
}
......
AngularApp/prototype/src/app/table-overview/table-overview.component.ts
View file @ 8ca0800b
......@@ -73,10 +73,13 @@ export class TableOverviewComponent implements OnInit {
}
async createHistograms() {
console.log('creating table histograms');
this.subplots = [];
this.averages = [];
const tables = this.state.table;
tables.forEach((table) => {
console.log('start of table histograms');
tables.forEach((table, index) => {
console.log(index);
this.subplots.push(
{
data: [{
......@@ -112,6 +115,7 @@ export class TableOverviewComponent implements OnInit {
}
);
});
console.log('tables histogram created');
}
// async setQuery(data) {
......
Flaskserver/.idea/workspace.xml
View file @ 8ca0800b
......@@ -21,24 +21,14 @@
<component name="ChangeListManager">
<list default="true" id="556080ba-825c-4b55-a92a-867a4df4fb32" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/api.service.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/api.service.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/labeling-window/labeling-window.component.css" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/labeling-window/labeling-window.component.css" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/labeling-window/labeling-window.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/labeling-window/labeling-window.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/labels/labels.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/labels/labels.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.html" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.html" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/progress-view/progress-view.component.css" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/progress-view/progress-view.component.css" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/progress-view/progress-view.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/progress-view/progress-view.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/state.service.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/state.service.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/table-overview/table-overview.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/table-overview/table-overview.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/_lsh.cpython-38-x86_64-linux-gnu.so" beforeDir="false" afterPath="$PROJECT_DIR$/_lsh.cpython-38-x86_64-linux-gnu.so" afterDir="false" />
<change beforePath="$PROJECT_DIR$/build/lib.linux-x86_64-3.8/_lsh.cpython-38-x86_64-linux-gnu.so" beforeDir="false" afterPath="$PROJECT_DIR$/build/lib.linux-x86_64-3.8/_lsh.cpython-38-x86_64-linux-gnu.so" afterDir="false" />
<change beforePath="$PROJECT_DIR$/build/temp.linux-x86_64-3.8/locality-sensitive-hashing-visual-analytics/lsh-fast/_lsh.o" beforeDir="false" afterPath="$PROJECT_DIR$/build/temp.linux-x86_64-3.8/locality-sensitive-hashing-visual-analytics/lsh-fast/_lsh.o" afterDir="false" />
<change beforePath="$PROJECT_DIR$/build/temp.linux-x86_64-3.8/locality-sensitive-hashing-visual-analytics/lsh-fast/lsh.o" beforeDir="false" afterPath="$PROJECT_DIR$/build/temp.linux-x86_64-3.8/locality-sensitive-hashing-visual-analytics/lsh-fast/lsh.o" afterDir="false" />
<change beforePath="$PROJECT_DIR$/data.pkl" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/processed-data.npy" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/../experiments/.ipynb_checkpoints/MTS test-checkpoint.ipynb" beforeDir="false" afterPath="$PROJECT_DIR$/../experiments/.ipynb_checkpoints/MTS test-checkpoint.ipynb" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../experiments/MTS test.ipynb" beforeDir="false" afterPath="$PROJECT_DIR$/../experiments/MTS test.ipynb" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
......
Flaskserver/main.py
View file @ 8ca0800b
......@@ -110,6 +110,10 @@ def create_mts_windows():
windows.append(preprocessing.minmax_scale(data[i], (-1, 1), axis=1))
print("Preprocessed: " + str(time()-t0))
np.save('processed-data', windows)
# data = np.load('processed-data.npy')
# data = np.reshape(data, (len(data), len(data[0][0]), len(data[0])))
# r, a, sd = preprocess(data, 11.5)
# np.save('parameters', np.array([r, a, sd]))
print("Sending response: " + str(time()-t0))
return '1'
......@@ -174,8 +178,11 @@ def initialize():
data = np.reshape(data, (len(data), len(data[0][0]), len(data[0])))
query = raw_data["query"]
query = np.reshape(query, (len(query[0]), len(query)))
parameters = np.load('parameters.npy')
r = parameters[0]
a = parameters[1]
sd = parameters[2]
r, a, sd = preprocess(data)
candidates, distances, hf = _lsh.lsh(data, query, r, a, sd)
dict = defaultdict(int)
......@@ -187,9 +194,23 @@ def initialize():
average_candidates = list(sorted_dict.keys())
average_distances = list(sorted_dict.values())
tables = []
candidates = candidates.tolist()
for l in range(len(candidates)):
for k in range(len(candidates[0])):
dict = defaultdict(list)
length = len(distances[l][k])
median = distances[l][k][math.ceil(length/2)]
stepsize = median / 10
indices = list(map(lambda x: 19 if x > median * 2 else math.floor(x / stepsize), distances[l][k]))
for i in range(len(candidates[0][0])):
dict[str(indices[i])].append(candidates[l][k][i])
tables.append(dict)
response = {
"hash_functions": hf.tolist(),
"candidates": candidates.tolist(),
"candidates": tables,
"average_candidates": np.array(average_candidates).tolist(),
"average_distances": np.array(average_distances).tolist(),
"distances": distances.tolist(),
......@@ -222,10 +243,34 @@ def update():
parameters = raw_data["parameters"]
candidates, distances, hf = _lsh.lsh(data, query, parameters[0], parameters[1], parameters[2])
dict = defaultdict(int)
for l in range(len(candidates)):
for k in range(len(candidates[0])):
for i in range(len(candidates[0][0])):
dict[candidates[l][k][i]] += distances[l][k][i]
sorted_dict = {k: v for k, v in sorted(dict.items(), key=lambda item: item[1])}
average_candidates = list(sorted_dict.keys())
average_distances = list(sorted_dict.values())
tables = []
candidates = candidates.tolist()
for l in range(len(candidates)):
for k in range(len(candidates[0])):
dict = defaultdict(list)
length = len(distances[l][k])
median = distances[l][k][math.ceil(length/2)]
stepsize = median / 10
indices = list(map(lambda x: 19 if x > median * 2 else math.floor(x / stepsize), distances[l][k]))
for i in range(len(candidates[0][0])):
dict[str(indices[i])].append(candidates[l][k][i])
tables.append(dict)
response = {
"hash_functions": hf.tolist(),
"candidates": tables,
"average_candidates": np.array(average_candidates).tolist(),
"average_distances": np.array(average_distances).tolist(),
"distances": distances.tolist(),
"candidates": candidates.tolist()
}
response = orjson.dumps(response)
print('LSH done: ' + str(time()-t0))
......@@ -282,7 +327,7 @@ def table_info():
print("Averages calculated: " + str(time() - t0))
return response
def preprocess(data, r=1000):
def preprocess(data, r=10.0):
# return 0.10882589134534404, 3.1202154563478928, 0.9705780396843037
# data = np.load('processed-data.npy')
# data = np.reshape(data, (59999, 20, 120))
......@@ -293,6 +338,7 @@ def preprocess(data, r=1000):
i = 0
while i < len(data):
if i % 999 == 0:
print(r)
print(str(i) + ':' + str(len(subset)))
state = 1
......@@ -314,7 +360,7 @@ def preprocess(data, r=1000):
i = 0
# subset = sample(list(range(len(data))), 200)
print("r = " + str(r))
dtw_distances = []
eq_distances = []
for i, index_1 in enumerate(subset):
......@@ -363,7 +409,7 @@ def debug_test_lsh():
print(data.shape)
data = np.reshape(data, (len(data), len(data[0][0]), len(data[0])))
r, a, sd = preprocess(data, 10)
r, a, sd = preprocess(data, 11.25)
create_windows()
query_n = 1234
t0 = time()
......@@ -388,12 +434,16 @@ def debug_test_lsh():
print("Calculated exact dtw in: " + str(time()-t0))
print(topk_dtw[0:20])
t0 = time()
l2distances = [np.linalg.norm(window - query) for window in data]
print("Calculated exact l2 in: " + str(time()-t0))
# # distances_ed = [distance.euclidean(query, window) for window in data]
# # topk_ed = sorted(range(len(distances_ed)), key=lambda k: distances_ed[k])
#
accuracy = 0
for index in topk_dtw[0:20]:
if index in candidates[0:200]:
if index in candidates:
accuracy += 1
print(accuracy)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment