added retries to etcd and syncthing http calls

syncthing_monitor/etcd_client.py

@@ -1,6 +1,7 @@
 import json
 
 import etcd3 as etcd3
+from retrying import retry
 
 CLUSTER_INFO_KEY = '/syncthing_monitor/cluster_info'
 
@@ -10,6 +11,7 @@ class EtcdClient:
         self.etcd = etcd3.client(host=host, port=port)
         self.key = key
 
+    @retry(stop_max_delay=60000, wait_fixed=500)
     def load_cluster_info(self):
         print("Retrieving cluster info from etcd...", flush=True)
         raw_value = self.etcd.get(self.key)[0]

syncthing_monitor/syncthing_client.py

@@ -1,6 +1,7 @@
 import json
 
 import requests
+from retrying import retry
 
 
 class SyncthingClient:
@@ -137,11 +138,14 @@ class SyncthingClient:
     def make_url(self, endpoint):
         return "http://{0}:{1}{2}".format(self.host, self.port, endpoint)
 
+    @retry(stop_max_delay=60000, wait_fixed=500)
     def get(self, endpoint):
         return requests.get(self.make_url(endpoint), headers=self.headers)
 
+    @retry(stop_max_delay=60000, wait_fixed=500)
     def post(self, endpoint, data):
         return requests.post(self.make_url(endpoint), headers=self.headers, data=data)
 
+    @retry(stop_max_delay=60000, wait_fixed=500)
     def patch(self, endpoint, data):
         return requests.patch(self.make_url(endpoint), headers=self.headers, data=data)