From ac22c515f157c976d1d22e1751c03a861e7d4e15 Mon Sep 17 00:00:00 2001 From: James Snyder Date: Sun, 21 Apr 2013 21:13:42 -0500 Subject: [PATCH 1/6] simple attempt at getting more history --- goxapi.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/goxapi.py b/goxapi.py index 7e035f4..e8da21f 100644 --- a/goxapi.py +++ b/goxapi.py @@ -42,6 +42,8 @@ import logging import Queue import time +import datetime +import calendar import traceback import threading from urllib2 import Request as URLRequest @@ -651,7 +653,14 @@ def request_history(self): # Gox() will have set this field to the timestamp of the last # known candle, so we only request data since this time - since = self.history_last_candle + if self.history_last_candle: + since = self.history_last_candle + else: + endtime = datetime.datetime.now() + starttime = endtime - datetime.timedelta(days=3) + since = int(calendar.timegm(starttime.timetuple())) + + def history_thread(): """request trading history""" @@ -667,12 +676,19 @@ def history_thread(): self.debug("requesting history") use_ssl = self.config.get_bool("gox", "use_ssl") proto = {True: "https", False: "http"}[use_ssl] - json_hist = http_request(proto + "://" + self.HTTP_HOST \ - + "/api/2/BTC" + self.currency + "/money/trades" - + querystring) - history = json.loads(json_hist) + data = [] + while True: + json_hist = http_request(proto + "://" + self.HTTP_HOST \ + + "/api/2/BTC" + self.currency + "/money/trades" + + querystring) + history = json.loads(json_hist) + data += history["data"] + querystring = "?since=" + data[-1]["tid"] + print int(calendar.timegm(endtime.timetuple())) - int(data[-1]["date"]) + if int(data[-1]["date"]) > int(calendar.timegm(endtime.timetuple())): + break if history["result"] == "success": - self.signal_fullhistory(self, history["data"]) + self.signal_fullhistory(self, data) start_thread(history_thread) From 6f0cfde8ccdfd5d9bde704f4126fa328a7dd5e5d Mon Sep 17 00:00:00 2001 From: James Snyder Date: Wed, 24 Apr 2013 17:24:06 -0500 Subject: [PATCH 2/6] better handling of dates, finish grabbing if we hit the end of the feed --- goxapi.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/goxapi.py b/goxapi.py index e8da21f..1bc032e 100644 --- a/goxapi.py +++ b/goxapi.py @@ -43,7 +43,6 @@ import Queue import time import datetime -import calendar import traceback import threading from urllib2 import Request as URLRequest @@ -653,12 +652,12 @@ def request_history(self): # Gox() will have set this field to the timestamp of the last # known candle, so we only request data since this time + endtime = datetime.datetime.now() if self.history_last_candle: since = self.history_last_candle else: - endtime = datetime.datetime.now() - starttime = endtime - datetime.timedelta(days=3) - since = int(calendar.timegm(starttime.timetuple())) + starttime = endtime - datetime.timedelta(days=2) + since = int(time.mktime(starttime.timetuple())) @@ -682,11 +681,13 @@ def history_thread(): + "/api/2/BTC" + self.currency + "/money/trades" + querystring) history = json.loads(json_hist) + if len(history["data"]) == 0: + break data += history["data"] querystring = "?since=" + data[-1]["tid"] - print int(calendar.timegm(endtime.timetuple())) - int(data[-1]["date"]) - if int(data[-1]["date"]) > int(calendar.timegm(endtime.timetuple())): - break + self.debug("seconds behind: %d" % (int(time.mktime(endtime.timetuple())) - int(data[-1]["date"]))) + if int(data[-1]["date"]) > int(time.mktime(endtime.timetuple())): + break if history["result"] == "success": self.signal_fullhistory(self, data) From a3b7b2d4071428d3d2e4dffc1d27c63ce1fb5506 Mon Sep 17 00:00:00 2001 From: James Snyder Date: Tue, 30 Apr 2013 11:30:22 -0500 Subject: [PATCH 3/6] minor fixes --- goxapi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/goxapi.py b/goxapi.py index 94880f8..d02a7b9 100644 --- a/goxapi.py +++ b/goxapi.py @@ -672,10 +672,10 @@ def request_history(self): if self.history_last_candle: since = self.history_last_candle else: - days = config.get_int("gox", "history_length_days") + days = self.config.get_int("gox", "history_length_days") if not days: days = 2 - starttime = endtime - datetime.timedelta(days=days) + starttime = datetime.datetime.now() - datetime.timedelta(days=days) since = int(time.mktime(starttime.timetuple())) From e610b5b637f309b90d544ef0fc92232e178e9b91 Mon Sep 17 00:00:00 2001 From: James Snyder Date: Thu, 9 May 2013 12:23:21 -0500 Subject: [PATCH 4/6] change notification message for downloads, set endtime earlier since we still get live trades --- goxapi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/goxapi.py b/goxapi.py index 1677728..30fb7cc 100644 --- a/goxapi.py +++ b/goxapi.py @@ -722,6 +722,7 @@ def request_history(self): # Gox() will have set this field to the timestamp of the last # known candle, so we only request data since this time + endtime = datetime.datetime.now() if self.history_last_candle: since = self.history_last_candle else: @@ -749,7 +750,6 @@ def history_thread(): proto = {True: "https", False: "http"}[use_ssl] data = [] while True: - endtime = datetime.datetime.now() json_hist = http_request(proto + "://" + HTTP_HOST \ + "/api/2/" + self.curr_base + self.curr_quote \ + "/money/trades" + querystring) @@ -758,7 +758,7 @@ def history_thread(): break data += history["data"] querystring = "?since=" + data[-1]["tid"] - self.debug("seconds behind: %d" % (int(time.mktime(endtime.timetuple())) - int(data[-1]["date"]))) + self.debug("download: %d (seconds behind)" % (int(time.mktime(endtime.timetuple())) - int(data[-1]["date"]))) if int(data[-1]["date"]) > int(time.mktime(endtime.timetuple())): break if history["result"] == "success": From 363993edb28c13a086184e5891d3b355d9b3a0fa Mon Sep 17 00:00:00 2001 From: James Snyder Date: Mon, 13 May 2013 17:07:18 -0500 Subject: [PATCH 5/6] display in days rather than seconds --- goxapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/goxapi.py b/goxapi.py index 30fb7cc..1edd4d7 100644 --- a/goxapi.py +++ b/goxapi.py @@ -758,7 +758,7 @@ def history_thread(): break data += history["data"] querystring = "?since=" + data[-1]["tid"] - self.debug("download: %d (seconds behind)" % (int(time.mktime(endtime.timetuple())) - int(data[-1]["date"]))) + self.debug("download: %2.2f (days behind)" % ((time.mktime(endtime.timetuple()) - data[-1]["date"])/86400)) if int(data[-1]["date"]) > int(time.mktime(endtime.timetuple())): break if history["result"] == "success": From 7c141065bf1ae03f248e6f0923076693873a2dc5 Mon Sep 17 00:00:00 2001 From: James Snyder Date: Wed, 15 May 2013 13:46:59 -0500 Subject: [PATCH 6/6] deal with bad responses to history requests --- goxapi.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/goxapi.py b/goxapi.py index fc4fdc6..ba5c75c 100644 --- a/goxapi.py +++ b/goxapi.py @@ -748,21 +748,26 @@ def history_thread(): proto = {True: "https", False: "http"}[use_ssl] data = [] while True: - json_hist = http_request("%s://%s/api/2/%s%s/money/trades%s" % ( - proto, - HTTP_HOST, - self.curr_base, - self.curr_quote, - querystring - )) - history = json.loads(json_hist) - if len(history["data"]) == 0: - break - data += history["data"] - querystring = "?since=" + data[-1]["tid"] - self.debug("download: %2.2f (days behind)" % ((time.mktime(endtime.timetuple()) - data[-1]["date"])/86400)) - if int(data[-1]["date"]) > int(time.mktime(endtime.timetuple())): - break + try: + json_hist = http_request("%s://%s/api/2/%s%s/money/trades%s" % ( + proto, + HTTP_HOST, + self.curr_base, + self.curr_quote, + querystring + )) + history = json.loads(json_hist) + if len(history["data"]) == 0: + break + data += history["data"] + querystring = "?since=" + data[-1]["tid"] + self.debug("download: %2.2f (days behind)" % ((time.mktime(endtime.timetuple()) - data[-1]["date"])/86400)) + if int(data[-1]["date"]) > int(time.mktime(endtime.timetuple())): + break + except ValueError: + self.debug("download: bad response, retrying") + time.sleep(1) + if history["result"] == "success": self.signal_fullhistory(self, data)