fix date parsing issue

DatastoreServer/GrafanaDatastoreServer.py

@@ -3,16 +3,34 @@
 import argparse
 import redis
 import flask
-import calendar
+import pytz
+from datetime import timedelta, datetime
 import dateutil.parser
-from gevent.wsgi import WSGIServer
+from gevent.pywsgi import WSGIServer
 from flask import Flask, jsonify
 from flask_cors import CORS, cross_origin
 
 app = Flask(__name__)
 CORS(app)
 
+timezone = pytz.timezone("UTC")
+EPOCH = timezone.localize(datetime(1970, 1, 1, 0, 0, 0))
+
 REDIS_POOL = None
+SCAN_TYPE_SCRIPT = """local cursor, pat, typ, cnt = ARGV[1], ARGV[2], ARGV[3], ARGV[4] or 100
+local rep = {}
+
+local res = redis.call('SCAN', cursor, 'MATCH', pat, 'COUNT', cnt)
+while #res[2] > 0 do
+  local k = table.remove(res[2])
+  local t = redis.call('TYPE', k)
+  if t['ok'] == typ then
+    table.insert(rep, k)
+  end
+end
+
+rep = {tonumber(res[1]), rep}
+return rep"""
 
 @app.route('/')
 @cross_origin()
@@ -23,13 +41,21 @@ def hello_world():
 @cross_origin()
 def search():
     redis_client = redis.Redis(connection_pool=REDIS_POOL)
-    return jsonify(redis_client.keys())
+    result = []
+    cursor = 0
+    while True:
+        cursor, keys = redis_client.eval(SCAN_TYPE_SCRIPT, 0, cursor, "*", "TSDB-TYPE", 100)
+        result.extend([k.decode("ascii") for k in keys])
+        if cursor == 0:
+            break
+
+    return jsonify(result)
 
 def process_targets(targets, redis_client):
     result = []
     for target in targets:
         if '*' in target:
-            result.extend(redis_client.keys(target))
+            result.extend([k.decode('ascii') for k in redis_client.keys(target)])
         else:
             result.append(target)
     return result
@@ -39,19 +65,24 @@ def query():
     request = flask.request.get_json()
     response = []
 
-    stime = calendar.timegm(dateutil.parser.parse(request['range']['from']).timetuple())
-    etime = calendar.timegm(dateutil.parser.parse(request['range']['to']).timetuple())
+    # !!! dates 'from' and 'to' are expected to be in UTC, which is what Grafana provides here.
+    # If not in UTC, use pytz to set to UTC timezone and subtract the utcoffset().
+    # Time delta calculations should always be done in UTC to avoid pitfalls of daylight offset changes.
+    stime = (dateutil.parser.parse(request['range']['from']) - EPOCH).total_seconds() * 1000
+    etime = (dateutil.parser.parse(request['range']['to']) - EPOCH).total_seconds() * 1000
+    
+    
 
     redis_client = redis.Redis(connection_pool=REDIS_POOL)
     targets = process_targets([t['target'] for t in request['targets']], redis_client)
 
     for target in targets:
         args = ['ts.range', target, int(stime), int(etime)]
-        if 'intervalMs' in request and request['intervalMs'] > 0 and request['intervalMs']/1000 > 1:
-            args += ['avg', int(round(request['intervalMs']/1000))]
+        if 'intervalMs' in request and request['intervalMs'] > 0:
+            args += ['avg', int(request['intervalMs'])]
         print(args)
         redis_resp = redis_client.execute_command(*args)
-        datapoints = [(x2.decode("ascii"), x1*1000) for x1, x2 in redis_resp]
+        datapoints = [(float(x2.decode("ascii")), x1) for x1, x2 in redis_resp]
         response.append(dict(target=target, datapoints=datapoints))
     return jsonify(response)
 

DatastoreServer/requirements.txt

@@ -1,9 +1,6 @@
-git+git://github.com/RedisTimeSeries/redistimeseries-py
-hiredis>=0.2.0
-redis>=2.10
-rmtest>=0.2
-six>=1.10.0
-gevent==1.2.1
+redis==2.10.5
+gevent>=1.3.a1
 Flask>=0.12
 Flask-Cors==3.0.2
 python-dateutil==2.6.0
+pytz==2019.2