Add progress view (incomplete) + store windows server side (faster)

Flaskserver/main.py

@@ -8,6 +8,7 @@ import os.path
 import json
 from sklearn import preprocessing
 import orjson
+import dask.dataframe as dd
 
 app = Flask(__name__)
 CORS(app)
@@ -18,16 +19,16 @@ def index():
 
 @app.route('/read-data', methods=['GET'])
 def read_data():
-    filename = 'processed-data.pkl'
-    # if (not os.path.isfile(filename)):
-    #     print("start")
-    #     df = dd.read_csv("NW_Ground_Stations_2016.csv", usecols=['number_sta', 'date', 't'])
-    #     print("read file")
-    #     df = df.loc[df['number_sta'] == 14066001]
-    #     print("split rows")
-    #     df = df.compute()
-    #     df.to_pickle(filename)
-    #     print("to_pandas")
+    filename = 'data.pkl'
+    if (not os.path.isfile(filename)):
+        print("start")
+        df = dd.read_csv("NW_Ground_Stations_2016.csv", usecols=['number_sta', 'date', 't'])
+        print("read file")
+        df = df.loc[df['number_sta'] == 14066001]
+        print("split rows")
+        df = df.compute()
+        df.to_pickle(filename)
+        print("to_pandas")
     df = pd.read_pickle(filename)
     df.dropna(subset=['t'], inplace=True)
     response = {
@@ -38,49 +39,37 @@ def read_data():
     response = jsonify(response)
     return response
 
-# @app.route('/read-data', methods=['GET'])
-# def read_data():
-#     df = pd.read_csv("1.csv", index_col=3)
-#     df.index = pd.to_datetime(df.index)
-#     df.sort_index(inplace=True)
-#     meantemp = df.loc[:, 7].copy()
-#     response = {
-#         "index": meantemp.index.values.astype(str).tolist(),
-#         "values": meantemp.values.tolist()
-#     }
-#     response = jsonify(response)
-#     return response
-
 @app.route('/create-windows', methods=['POST'])
 def create_windows():
     t0 = time()
+    if (not os.path.isfile('processed-data.npy')):
+        filename = 'data.pkl'
+        df = pd.read_pickle(filename)
+        values = df.loc[:, 't'].values.astype(str).tolist()
+        print("Data read: " + str(time()-t0))
         raw_data = request.json
-    values = raw_data["values"]
         window_size = int(raw_data['parameters']["windowsize"])
+        print("Processing: " + str(time()-t0))
         data = [values[i:i+window_size] for i in range(len(values) - window_size)]
         data = preprocessing.minmax_scale(data, (-1, 1), axis=1)
-    print("Created windows: " + str(time()-t0))
-    data = data.tolist()
-    print("data converted: " + str(time()-t0))
-    # response = {'data': data}
+        print("Preprocessed: " + str(time()-t0))
+        np.save('processed-data', data)
     print("Sending response: " + str(time()-t0))
-    response = orjson.dumps(data)
-    print("Sending response: " + str(time()-t0))
-    return response
+    return '1'
 
 @app.route('/create-tables', methods=['POST'])
 def create_tables():
     t0 = time()
+    print("loading")
+    data = np.load('processed-data.npy')
+    print(time()-t0)
     raw_data = orjson.loads(request.data)
     print(time()-t0)
-    global data
-    data = raw_data["windows"]
     window_size = int(raw_data['parameters']["windowsize"])
     hash_size = int(raw_data['parameters']["hashsize"])
     table_size = int(raw_data['parameters']["tablesize"])
     data = np.array(data)
     print('Starting: ' + str(time()-t0))
-    global tables_hash_function
     tables_hash_function = [np.random.uniform(-1, 1, size=(window_size, hash_size)) for _ in range(table_size)]
     print('Init time: ' + str(time() - t0))
     tables = []
@@ -107,10 +96,13 @@ def create_tables():
 
 @app.route('/query', methods=['POST'])
 def query():
+    t0 = time()
     raw_data = orjson.loads(request.data)
-    window = raw_data["window"]
+    query = raw_data['query']
     tables = raw_data["tables"]
     neighbours = []
+    data = np.load('processed-data.npy')
+    window = data[query]
 
     output = defaultdict(list)
 
@@ -121,15 +113,33 @@ def query():
     for index, frequency in neighbours_with_frequency.items():
         output[str(frequency)].append(index)
     response = orjson.dumps(output)
+    print("Query done: " + str(time()-t0))
+    return response
+
+@app.route('/average', methods=['POST'])
+def average():
+    t0 = time()
+    raw_data = orjson.loads(request.data)
+    all_windows = raw_data['windows']
+    data = np.load('processed-data.npy')
+    output = []
+    for windows in all_windows:
+        t1 = time()
+        actual_windows = data[windows]
+        output.append((np.sum(actual_windows, 0)/len(actual_windows)).tolist())
+        #output.append([(sum(x)/len(actual_windows)).item() for x in zip(*actual_windows)])
+        print("Average calculated: " + str(time() - t1))
+    response = orjson.dumps(output)
+    print("Average calculated: " + str(time() - t0))
     return response
 
 @app.route('/update', methods=['POST'])
 def update():
     t0 = time()
+    print("Start")
     raw_data = orjson.loads(request.data)
-
-    data = raw_data["windows"]
-    data = np.array(data)
+    print("Data loaded: " + str(time() - t0))
+    data = np.load('processed-data.npy')
     label_data = raw_data["labelData"]
     tables = raw_data["tables"]
 
@@ -163,11 +173,11 @@ def update():
         t1 = time()
         while True:
             hash_function = np.random.randn(window_size, hash_size)
-            correct_signatures = [''.join((np.dot(data[index], hash_function) > 0).astype('int').astype('str')) for
-                                  index in
+            correct_signatures = [''.join((np.dot(data[i], hash_function) > 0).astype('int').astype('str')) for
+                                  i in
                                   correct_indices]
-            incorrect_signatures = [''.join((np.dot(data[index], hash_function) > 0).astype('int').astype('str')) for
-                                    index
+            incorrect_signatures = [''.join((np.dot(data[i], hash_function) > 0).astype('int').astype('str')) for
+                                    i
                                     in incorrect_indices]
             if correct_signatures.count(correct_signatures[0]) == len(
                     correct_signatures) and incorrect_signatures.count(

Flaskserver/venv/Lib/site-packages/Flask-1.1.2.dist-info/RECORD

-../../Scripts/flask.exe,sha256=KStO2ltsP8-uLy5Z9oRVvO1lsShiehcMI9shoHyJTUY,103300
+../../Scripts/flask.exe,sha256=tXDH2yw_IVFYBuQ5xdTtRi0Kra-qpPi8YOM5gicfUtA,103300
 Flask-1.1.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
 Flask-1.1.2.dist-info/LICENSE.rst,sha256=SJqOEQhQntmKN7uYPhHg9-HTHwvY-Zp5yESOf_N9B-o,1475
 Flask-1.1.2.dist-info/METADATA,sha256=3INpPWH6nKfZ33R2N-bQZy4TOe1wQCMweZc9mwcNrtc,4591

Flaskserver/venv/Lib/site-packages/Flask_Cors-3.0.8.dist-info/INSTALLER

-pip

Flaskserver/venv/Lib/site-packages/Flask_Cors-3.0.8.dist-info/LICENSE

-Copyright (C) 2016 Cory Dolphin, Olin College
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
\ No newline at end of file

Flaskserver/venv/Lib/site-packages/Flask_Cors-3.0.8.dist-info/METADATA

-Metadata-Version: 2.1
-Name: Flask-Cors
-Version: 3.0.8
-Summary: A Flask extension adding a decorator for CORS support
-Home-page: https://github.com/corydolphin/flask-cors
-Author: Cory Dolphin
-Author-email: corydolphin@gmail.com
-License: MIT
-Platform: any
-Classifier: Environment :: Web Environment
-Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 2
-Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.4
-Classifier: Programming Language :: Python :: 3.5
-Classifier: Programming Language :: Python :: 3.6
-Classifier: Programming Language :: Python :: 3.7
-Classifier: Programming Language :: Python :: Implementation :: CPython
-Classifier: Programming Language :: Python :: Implementation :: PyPy
-Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
-Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Dist: Flask (>=0.9)
-Requires-Dist: Six
-
-Flask-CORS
-==========
-
-|Build Status| |Latest Version| |Supported Python versions|
-|License|
-
-A Flask extension for handling Cross Origin Resource Sharing (CORS),
-making cross-origin AJAX possible.
-
-This package has a simple philosophy, when you want to enable CORS, you
-wish to enable it for all use cases on a domain. This means no mucking
-around with different allowed headers, methods, etc. By default,
-submission of cookies across domains is disabled due to the security
-implications, please see the documentation for how to enable
-credential'ed requests, and please make sure you add some sort of
-`CSRF <http://en.wikipedia.org/wiki/Cross-site_request_forgery>`__
-protection before doing so!
-
-Installation
-------------
-
-Install the extension with using pip, or easy\_install.
-
-.. code:: bash
-
-    $ pip install -U flask-cors
-
-Usage
------
-
-This package exposes a Flask extension which by default enables CORS support on all routes, for all origins and methods. It allows parameterization of all CORS headers on a per-resource level. The package also contains a decorator, for those who prefer this approach.
-
-Simple Usage
-~~~~~~~~~~~~
-
-In the simplest case, initialize the Flask-Cors extension with default
-arguments in order to allow CORS for all domains on all routes. See the
-full list of options in the `documentation <https://flask-cors.corydolphin.com/en/latest/api.html#extension>`__.
-
-.. code:: python
-
-
-    from flask import Flask
-    from flask_cors import CORS
-
-    app = Flask(__name__)
-    CORS(app)
-
-    @app.route("/")
-    def helloWorld():
-      return "Hello, cross-origin-world!"
-
-Resource specific CORS
-^^^^^^^^^^^^^^^^^^^^^^
-
-Alternatively, you can specify CORS options on a resource and origin
-level of granularity by passing a dictionary as the `resources` option,
-mapping paths to a set of options. See the
-full list of options in the `documentation <https://flask-cors.corydolphin.com/en/latest/api.html#extension>`__.
-
-.. code:: python
-
-    app = Flask(__name__)
-    cors = CORS(app, resources={r"/api/*": {"origins": "*"}})
-
-    @app.route("/api/v1/users")
-    def list_users():
-      return "user example"
-
-Route specific CORS via decorator
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-This extension also exposes a simple decorator to decorate flask routes
-with. Simply add ``@cross_origin()`` below a call to Flask's
-``@app.route(..)`` to allow CORS on a given route. See the
-full list of options in the `decorator documentation <https://flask-cors.corydolphin.com/en/latest/api.html#decorator>`__.
-
-.. code:: python
-
-    @app.route("/")
-    @cross_origin()
-    def helloWorld():
-      return "Hello, cross-origin-world!"
-
-Documentation
--------------
-
-For a full list of options, please see the full
-`documentation <https://flask-cors.corydolphin.com/en/latest/api.html>`__
-
-Troubleshooting
----------------
-
-If things aren't working as you expect, enable logging to help understand
-what is going on under the hood, and why.
-
-.. code:: python
-
-    logging.getLogger('flask_cors').level = logging.DEBUG
-
-
-Tests
------
-
-A simple set of tests is included in ``test/``. To run, install nose,
-and simply invoke ``nosetests`` or ``python setup.py test`` to exercise
-the tests.
-
-Contributing
-------------
-
-Questions, comments or improvements? Please create an issue on
-`Github <https://github.com/corydolphin/flask-cors>`__, tweet at
-`@corydolphin <https://twitter.com/corydolphin>`__ or send me an email.
-I do my best to include every contribution proposed in any way that I
-can.
-
-Credits
--------
-
-This Flask extension is based upon the `Decorator for the HTTP Access
-Control <http://flask.pocoo.org/snippets/56/>`__ written by Armin
-Ronacher.
-
-.. |Build Status| image:: https://api.travis-ci.org/corydolphin/flask-cors.svg?branch=master
-   :target: https://travis-ci.org/corydolphin/flask-cors
-.. |Latest Version| image:: https://img.shields.io/pypi/v/Flask-Cors.svg
-   :target: https://pypi.python.org/pypi/Flask-Cors/
-.. |Supported Python versions| image:: https://img.shields.io/pypi/pyversions/Flask-Cors.svg
-   :target: https://img.shields.io/pypi/pyversions/Flask-Cors.svg
-.. |License| image:: http://img.shields.io/:license-mit-blue.svg
-   :target: https://pypi.python.org/pypi/Flask-Cors/
-
-

Flaskserver/venv/Lib/site-packages/Flask_Cors-3.0.8.dist-info/RECORD

-Flask_Cors-3.0.8.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
-Flask_Cors-3.0.8.dist-info/LICENSE,sha256=bhob3FSDTB4HQMvOXV9vLK4chG_Sp_SCsRZJWU-vvV0,1069
-Flask_Cors-3.0.8.dist-info/METADATA,sha256=WAcJQwYw0qqy43tjaa9SBNOYPCyzqayKZZwJ_RLRSi0,5368
-Flask_Cors-3.0.8.dist-info/RECORD,,
-Flask_Cors-3.0.8.dist-info/WHEEL,sha256=h_aVn5OB2IERUjMbi2pucmR_zzWJtk303YXvhh60NJ8,110
-Flask_Cors-3.0.8.dist-info/top_level.txt,sha256=aWye_0QNZPp_QtPF4ZluLHqnyVLT9CPJsfiGhwqkWuo,11
-flask_cors/__init__.py,sha256=oJExwfR7yU3HAsmQ_EfL6KoLK3zq3J9HsET9r-56sfM,791
-flask_cors/__pycache__/__init__.cpython-38.pyc,,
-flask_cors/__pycache__/core.cpython-38.pyc,,
-flask_cors/__pycache__/decorator.cpython-38.pyc,,
-flask_cors/__pycache__/extension.cpython-38.pyc,,
-flask_cors/__pycache__/version.cpython-38.pyc,,
-flask_cors/core.py,sha256=ZzJ4yoAhgbt-xVBqFFwsDDCzKbiyKL7PlufoVByq9ro,13895
-flask_cors/decorator.py,sha256=iiwjUi0lVeCm4OJJHY5Cvuzj2nENbUns3Iq6zqKXuss,4937
-flask_cors/extension.py,sha256=FsHOkMSMj5Zv0u_I0fgVdLh0QDUpmEvt_8Xw0qVBK-8,7427
-flask_cors/version.py,sha256=sF9np2e_SuSeS9bX2MWnkn45e-mKQh1f9lVU_kod4Yg,22

Flaskserver/venv/Lib/site-packages/Flask_Cors-3.0.8.dist-info/WHEEL

-Wheel-Version: 1.0
-Generator: bdist_wheel (0.33.4)
-Root-Is-Purelib: true
-Tag: py2-none-any
-Tag: py3-none-any
-

Flaskserver/venv/Lib/site-packages/Flask_Cors-3.0.8.dist-info/top_level.txt

-flask_cors