Commit 1fee93d4 authored by Kruyff,D.L.W. (Dylan)'s avatar Kruyff,D.L.W. (Dylan)
Browse files

Add progress view (incomplete) + store windows server side (faster)

parent b41c932e
......@@ -8,6 +8,7 @@ import os.path
import json
from sklearn import preprocessing
import orjson
import dask.dataframe as dd
app = Flask(__name__)
CORS(app)
......@@ -18,16 +19,16 @@ def index():
@app.route('/read-data', methods=['GET'])
def read_data():
filename = 'processed-data.pkl'
# if (not os.path.isfile(filename)):
# print("start")
# df = dd.read_csv("NW_Ground_Stations_2016.csv", usecols=['number_sta', 'date', 't'])
# print("read file")
# df = df.loc[df['number_sta'] == 14066001]
# print("split rows")
# df = df.compute()
# df.to_pickle(filename)
# print("to_pandas")
filename = 'data.pkl'
if (not os.path.isfile(filename)):
print("start")
df = dd.read_csv("NW_Ground_Stations_2016.csv", usecols=['number_sta', 'date', 't'])
print("read file")
df = df.loc[df['number_sta'] == 14066001]
print("split rows")
df = df.compute()
df.to_pickle(filename)
print("to_pandas")
df = pd.read_pickle(filename)
df.dropna(subset=['t'], inplace=True)
response = {
......@@ -38,49 +39,37 @@ def read_data():
response = jsonify(response)
return response
# @app.route('/read-data', methods=['GET'])
# def read_data():
# df = pd.read_csv("1.csv", index_col=3)
# df.index = pd.to_datetime(df.index)
# df.sort_index(inplace=True)
# meantemp = df.loc[:, 7].copy()
# response = {
# "index": meantemp.index.values.astype(str).tolist(),
# "values": meantemp.values.tolist()
# }
# response = jsonify(response)
# return response
@app.route('/create-windows', methods=['POST'])
def create_windows():
t0 = time()
if (not os.path.isfile('processed-data.npy')):
filename = 'data.pkl'
df = pd.read_pickle(filename)
values = df.loc[:, 't'].values.astype(str).tolist()
print("Data read: " + str(time()-t0))
raw_data = request.json
values = raw_data["values"]
window_size = int(raw_data['parameters']["windowsize"])
print("Processing: " + str(time()-t0))
data = [values[i:i+window_size] for i in range(len(values) - window_size)]
data = preprocessing.minmax_scale(data, (-1, 1), axis=1)
print("Created windows: " + str(time()-t0))
data = data.tolist()
print("data converted: " + str(time()-t0))
# response = {'data': data}
print("Preprocessed: " + str(time()-t0))
np.save('processed-data', data)
print("Sending response: " + str(time()-t0))
response = orjson.dumps(data)
print("Sending response: " + str(time()-t0))
return response
return '1'
@app.route('/create-tables', methods=['POST'])
def create_tables():
t0 = time()
print("loading")
data = np.load('processed-data.npy')
print(time()-t0)
raw_data = orjson.loads(request.data)
print(time()-t0)
global data
data = raw_data["windows"]
window_size = int(raw_data['parameters']["windowsize"])
hash_size = int(raw_data['parameters']["hashsize"])
table_size = int(raw_data['parameters']["tablesize"])
data = np.array(data)
print('Starting: ' + str(time()-t0))
global tables_hash_function
tables_hash_function = [np.random.uniform(-1, 1, size=(window_size, hash_size)) for _ in range(table_size)]
print('Init time: ' + str(time() - t0))
tables = []
......@@ -107,10 +96,13 @@ def create_tables():
@app.route('/query', methods=['POST'])
def query():
t0 = time()
raw_data = orjson.loads(request.data)
window = raw_data["window"]
query = raw_data['query']
tables = raw_data["tables"]
neighbours = []
data = np.load('processed-data.npy')
window = data[query]
output = defaultdict(list)
......@@ -121,15 +113,33 @@ def query():
for index, frequency in neighbours_with_frequency.items():
output[str(frequency)].append(index)
response = orjson.dumps(output)
print("Query done: " + str(time()-t0))
return response
@app.route('/average', methods=['POST'])
def average():
t0 = time()
raw_data = orjson.loads(request.data)
all_windows = raw_data['windows']
data = np.load('processed-data.npy')
output = []
for windows in all_windows:
t1 = time()
actual_windows = data[windows]
output.append((np.sum(actual_windows, 0)/len(actual_windows)).tolist())
#output.append([(sum(x)/len(actual_windows)).item() for x in zip(*actual_windows)])
print("Average calculated: " + str(time() - t1))
response = orjson.dumps(output)
print("Average calculated: " + str(time() - t0))
return response
@app.route('/update', methods=['POST'])
def update():
t0 = time()
print("Start")
raw_data = orjson.loads(request.data)
data = raw_data["windows"]
data = np.array(data)
print("Data loaded: " + str(time() - t0))
data = np.load('processed-data.npy')
label_data = raw_data["labelData"]
tables = raw_data["tables"]
......@@ -163,11 +173,11 @@ def update():
t1 = time()
while True:
hash_function = np.random.randn(window_size, hash_size)
correct_signatures = [''.join((np.dot(data[index], hash_function) > 0).astype('int').astype('str')) for
index in
correct_signatures = [''.join((np.dot(data[i], hash_function) > 0).astype('int').astype('str')) for
i in
correct_indices]
incorrect_signatures = [''.join((np.dot(data[index], hash_function) > 0).astype('int').astype('str')) for
index
incorrect_signatures = [''.join((np.dot(data[i], hash_function) > 0).astype('int').astype('str')) for
i
in incorrect_indices]
if correct_signatures.count(correct_signatures[0]) == len(
correct_signatures) and incorrect_signatures.count(
......
../../Scripts/flask.exe,sha256=KStO2ltsP8-uLy5Z9oRVvO1lsShiehcMI9shoHyJTUY,103300
../../Scripts/flask.exe,sha256=tXDH2yw_IVFYBuQ5xdTtRi0Kra-qpPi8YOM5gicfUtA,103300
Flask-1.1.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
Flask-1.1.2.dist-info/LICENSE.rst,sha256=SJqOEQhQntmKN7uYPhHg9-HTHwvY-Zp5yESOf_N9B-o,1475
Flask-1.1.2.dist-info/METADATA,sha256=3INpPWH6nKfZ33R2N-bQZy4TOe1wQCMweZc9mwcNrtc,4591
......
Copyright (C) 2016 Cory Dolphin, Olin College
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
\ No newline at end of file
Metadata-Version: 2.1
Name: Flask-Cors
Version: 3.0.8
Summary: A Flask extension adding a decorator for CORS support
Home-page: https://github.com/corydolphin/flask-cors
Author: Cory Dolphin
Author-email: corydolphin@gmail.com
License: MIT
Platform: any
Classifier: Environment :: Web Environment
Classifier: Intended Audience :: Developers
Classifier: License :: OSI Approved :: MIT License
Classifier: Operating System :: OS Independent
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 2
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.4
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy
Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
Classifier: Topic :: Software Development :: Libraries :: Python Modules
Requires-Dist: Flask (>=0.9)
Requires-Dist: Six
Flask-CORS
==========
|Build Status| |Latest Version| |Supported Python versions|
|License|
A Flask extension for handling Cross Origin Resource Sharing (CORS),
making cross-origin AJAX possible.
This package has a simple philosophy, when you want to enable CORS, you
wish to enable it for all use cases on a domain. This means no mucking
around with different allowed headers, methods, etc. By default,
submission of cookies across domains is disabled due to the security
implications, please see the documentation for how to enable
credential'ed requests, and please make sure you add some sort of
`CSRF <http://en.wikipedia.org/wiki/Cross-site_request_forgery>`__
protection before doing so!
Installation
------------
Install the extension with using pip, or easy\_install.
.. code:: bash
$ pip install -U flask-cors
Usage
-----
This package exposes a Flask extension which by default enables CORS support on all routes, for all origins and methods. It allows parameterization of all CORS headers on a per-resource level. The package also contains a decorator, for those who prefer this approach.
Simple Usage
~~~~~~~~~~~~
In the simplest case, initialize the Flask-Cors extension with default
arguments in order to allow CORS for all domains on all routes. See the
full list of options in the `documentation <https://flask-cors.corydolphin.com/en/latest/api.html#extension>`__.
.. code:: python
from flask import Flask
from flask_cors import CORS
app = Flask(__name__)
CORS(app)
@app.route("/")
def helloWorld():
return "Hello, cross-origin-world!"
Resource specific CORS
^^^^^^^^^^^^^^^^^^^^^^
Alternatively, you can specify CORS options on a resource and origin
level of granularity by passing a dictionary as the `resources` option,
mapping paths to a set of options. See the
full list of options in the `documentation <https://flask-cors.corydolphin.com/en/latest/api.html#extension>`__.
.. code:: python
app = Flask(__name__)
cors = CORS(app, resources={r"/api/*": {"origins": "*"}})
@app.route("/api/v1/users")
def list_users():
return "user example"
Route specific CORS via decorator
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
This extension also exposes a simple decorator to decorate flask routes
with. Simply add ``@cross_origin()`` below a call to Flask's
``@app.route(..)`` to allow CORS on a given route. See the
full list of options in the `decorator documentation <https://flask-cors.corydolphin.com/en/latest/api.html#decorator>`__.
.. code:: python
@app.route("/")
@cross_origin()
def helloWorld():
return "Hello, cross-origin-world!"
Documentation
-------------
For a full list of options, please see the full
`documentation <https://flask-cors.corydolphin.com/en/latest/api.html>`__
Troubleshooting
---------------
If things aren't working as you expect, enable logging to help understand
what is going on under the hood, and why.
.. code:: python
logging.getLogger('flask_cors').level = logging.DEBUG
Tests
-----
A simple set of tests is included in ``test/``. To run, install nose,
and simply invoke ``nosetests`` or ``python setup.py test`` to exercise
the tests.
Contributing
------------
Questions, comments or improvements? Please create an issue on
`Github <https://github.com/corydolphin/flask-cors>`__, tweet at
`@corydolphin <https://twitter.com/corydolphin>`__ or send me an email.
I do my best to include every contribution proposed in any way that I
can.
Credits
-------
This Flask extension is based upon the `Decorator for the HTTP Access
Control <http://flask.pocoo.org/snippets/56/>`__ written by Armin
Ronacher.
.. |Build Status| image:: https://api.travis-ci.org/corydolphin/flask-cors.svg?branch=master
:target: https://travis-ci.org/corydolphin/flask-cors
.. |Latest Version| image:: https://img.shields.io/pypi/v/Flask-Cors.svg
:target: https://pypi.python.org/pypi/Flask-Cors/
.. |Supported Python versions| image:: https://img.shields.io/pypi/pyversions/Flask-Cors.svg
:target: https://img.shields.io/pypi/pyversions/Flask-Cors.svg
.. |License| image:: http://img.shields.io/:license-mit-blue.svg
:target: https://pypi.python.org/pypi/Flask-Cors/
Flask_Cors-3.0.8.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
Flask_Cors-3.0.8.dist-info/LICENSE,sha256=bhob3FSDTB4HQMvOXV9vLK4chG_Sp_SCsRZJWU-vvV0,1069
Flask_Cors-3.0.8.dist-info/METADATA,sha256=WAcJQwYw0qqy43tjaa9SBNOYPCyzqayKZZwJ_RLRSi0,5368
Flask_Cors-3.0.8.dist-info/RECORD,,
Flask_Cors-3.0.8.dist-info/WHEEL,sha256=h_aVn5OB2IERUjMbi2pucmR_zzWJtk303YXvhh60NJ8,110
Flask_Cors-3.0.8.dist-info/top_level.txt,sha256=aWye_0QNZPp_QtPF4ZluLHqnyVLT9CPJsfiGhwqkWuo,11
flask_cors/__init__.py,sha256=oJExwfR7yU3HAsmQ_EfL6KoLK3zq3J9HsET9r-56sfM,791
flask_cors/__pycache__/__init__.cpython-38.pyc,,
flask_cors/__pycache__/core.cpython-38.pyc,,
flask_cors/__pycache__/decorator.cpython-38.pyc,,
flask_cors/__pycache__/extension.cpython-38.pyc,,
flask_cors/__pycache__/version.cpython-38.pyc,,
flask_cors/core.py,sha256=ZzJ4yoAhgbt-xVBqFFwsDDCzKbiyKL7PlufoVByq9ro,13895
flask_cors/decorator.py,sha256=iiwjUi0lVeCm4OJJHY5Cvuzj2nENbUns3Iq6zqKXuss,4937
flask_cors/extension.py,sha256=FsHOkMSMj5Zv0u_I0fgVdLh0QDUpmEvt_8Xw0qVBK-8,7427
flask_cors/version.py,sha256=sF9np2e_SuSeS9bX2MWnkn45e-mKQh1f9lVU_kod4Yg,22
Wheel-Version: 1.0
Generator: bdist_wheel (0.33.4)
Root-Is-Purelib: true
Tag: py2-none-any
Tag: py3-none-any
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment