Update for 2 LTEs

config/config.yaml

@@ -1,27 +1,31 @@
-database_migrate: False
-output_image_path: /var/www/downloads/speedgraph_{}.png
-output_txt_path: /var/www/downloads/speeds.txt
-ros_dynamic_speed: True
-ros_ip: 192.168.88.1
-ros_queues:
-  - all
-ros_queues_except:
-  - dummy
-ros_du_invert: False
-ros_fasttrack_comment: "defconf: fasttrack"
-# Minimum speed which not to cross, if speed is lower than this it will set this speed
-ros_minimum_speed: 100000
-# Speed at which to leave fasttrack on. Use if your ROS device's CPU can't handle max speed you get, for example LHG LTE
-# starts to show high CPU usage at close to 20mbit.
-# Not implemented yet
-ros_maximum_speed: 15000000 # 15 mbit
-ros_wan_interface:
-  - eth_LTE6_1
-  - eth_LTE_0
 api_host: 0.0.0.0
 api_port: 1357
 bad_speed_mbps: 0.2
+database_migrate: true
 good_speed_mbps: 8
+output_image_path: /var/www/downloads/speedgraph_{}.png
+output_txt_path: /var/www/downloads/speeds.txt
+ros_du_invert: True
+ros_dynamic_speed: true
+ros_fasttrack_comment: 'defconf: fasttrack'
+ros_ip: 192.168.88.1
+ros_maximum_speed: 15000000
+ros_minimum_speed: 100000
+ros_queues:
+  - queue1
+ros_queues_except:
+  - dummy
+ros_wan_interface:
+  - eth_LTE6_1
+  - eth_LTE_0
+ros_quality:
+  total:
+    - 1
+    - 20
+  eth_LTE6_1:
+    - 1
+    - 10
+  eth_LTE_0:
+    - 1
+    - 10
 
-
-# Conversion rate 1 mbit : 1000000 bits

dbo.py

@@ -45,6 +45,7 @@ class BroModel(Model):
 class Entry(BroModel):
     upload = FloatField(null=True)
     download = FloatField(null=True)
+    data = TextField(null=True)
 
 
     class Meta:
@@ -72,10 +73,10 @@ if config['database_migrate']:
 
         migrator = SqliteMigrator(db)
 
-        open_count = IntegerField(default=0)
+        data = TextField(null=True)
 
         migrate(
-            migrator.add_column('Entry', 'open_count', open_count)
+            migrator.add_column('Entry', 'data', data)
         )
         log.debug("Migration success")
         log.debug("=====================")
@@ -85,13 +86,15 @@ if config['database_migrate']:
     except:
         log.error("Could not migrate", exc_info=True)
         log.debug("=====================")
+else:
+    db.connect()
+    db.create_tables([Entry])
 # endregion
 
 log.info(" ".join(["Using DB", str(db), "At path:", str(db_path)]))
 
 # On init make sure we create database
 
-db.connect()
+
-db.create_tables([Entry])
 
 # endregion

speedtester.py

@@ -13,7 +13,6 @@ import speedtest
 from dbo import Entry
 import routeros_api
 
-
 config = read_config()
 secrets = read_config('secrets')
 
@@ -36,14 +35,16 @@ def gather_data():
         dates.append(entry.date_created)
 
     return dates, downloads, uploads
+
+
 import matplotlib
 import matplotlib.pyplot as plt
 
+
 @catch_errors
 def generate_plot_image(dates, downloads, uploads, name="speed", description="Speed Graph"):
     log.debug("Genering image output for {}...".format(name))
 
-
     dates = matplotlib.dates.date2num(dates)
     fig = plt.figure(figsize=(15, 3))
     plt.plot_date(dates, downloads, linestyle='solid', color="red", linewidth=2, marker=None)
@@ -59,7 +60,7 @@ def generate_diff(dates, downloads, uploads):
     up = []
     for i, date in enumerate(dates):
         curr = downloads[i] if downloads[i] else 0
-        prev = downloads[i-1] if downloads[i-1] and i > 0 else 0
+        prev = downloads[i - 1] if downloads[i - 1] and i > 0 else 0
         dl.append(curr - prev)
         curr = uploads[i] if uploads[i] else 0
         prev = uploads[i - 1] if uploads[i - 1] and i > 0 else 0
@@ -75,6 +76,7 @@ def generate_updown_plot_simple(downloads, uploads, name, description):
     plt.tight_layout()
     plt.savefig(read_config()['output_image_path'].format(name))
 
+
 @catch_errors
 def gather_day_median_data(dates, downloads, uploads):
     log.debug("Gather day median data...")
@@ -98,12 +100,14 @@ def gather_day_median_data(dates, downloads, uploads):
 
     return dl, up
 
+
 @catch_errors
 def generate_day_median(dates, downloads, uploads):
     dl, up = gather_day_median_data(dates, downloads, uploads)
-    
+
     generate_updown_plot_simple(dl, up, "day_median", "Day average")
 
+
 @catch_errors
 def generate_day_median_diff(dates, downloads, uploads):
     dl, up = gather_day_median_data(dates, downloads, uploads)
@@ -113,8 +117,8 @@ def generate_day_median_diff(dates, downloads, uploads):
 
     for i, down in enumerate(dl):
         if i > 0:
-            downs.append(dl[i]-dl[i-1])
+            downs.append(dl[i] - dl[i - 1])
-            ups.append(up[i]-up[i-1])
+            ups.append(up[i] - up[i - 1])
         else:
             downs.append(dl[i])
             ups.append(up[i])
@@ -161,6 +165,30 @@ def ros_fastrack_enable(enable):
     connection.disconnect()
 
 
+@catch_errors
+def ros_unlimited_speed():
+    connection = routeros_api.RouterOsApiPool(config['ros_ip'], username=secrets["ros_login"],
+                                              password=secrets["ros_password"], plaintext_login=True)
+    api = connection.get_api()
+
+    list_queues = api.get_resource('/queue/simple')
+    up = down = 4294967295
+    for queue in list_queues.get():
+        if queue['name'] in config['ros_queues'] or (
+                "all" in config['ros_queues'] and queue["name"] not in config['ros_queues_except']):
+            log.debug(f"Adjust Queue {queue['name']}: max_limit {int(up)}/{int(down)}")
+            try:
+                if config["ros_du_invert"] == True:
+                    # Inverting upload and download values, because master queue is most likely applied to the bridge
+                    list_queues.set(id=queue['id'], max_limit=f"{int(down)}/{int(up)}")
+                else:
+                    # Not inverting, use this in case master queue is applied to something like LTE interface
+                    list_queues.set(id=queue['id'], max_limit=f"{int(up)}/{int(down)}")
+            except:
+                log.error("Unable to change queue settings.", exc_info=True)
+    connection.disconnect()
+
+
 @catch_errors
 def ros_dynamic_speed(upload, download):
     '''
@@ -168,7 +196,7 @@ def ros_dynamic_speed(upload, download):
     :param upload: Upload speed from speedtest
     :param download: Download speed from speedtest
     '''
-    log.debug(f"Set Dynamic Speed to: DOWN {mbits(download)} mbps; UP {mbits(upload)} mbps")
+    log.debug(f"Set Speed to: DOWN {mbits(download)} mbps; UP {mbits(upload)} mbps")
     connection = routeros_api.RouterOsApiPool(config['ros_ip'], username=secrets["ros_login"],
                                               password=secrets["ros_password"], plaintext_login=True)
     api = connection.get_api()
@@ -176,21 +204,29 @@ def ros_dynamic_speed(upload, download):
     list_queues = api.get_resource('/queue/simple')
 
     for queue in list_queues.get():
-        if queue['name'] in config['ros_queues'] or ("all" in config['ros_queues'] and queue["name"] not in config['ros_queues_except']):
+        if queue['name'] in config['ros_queues'] or (
+                "all" in config['ros_queues'] and queue["name"] not in config['ros_queues_except']):
+            if queue['name'] in data_dict['wan_downloads']:
+                down = data_dict['wan_downloads'][queue['name']]
+                up = data_dict['wan_uploads'][queue['name']]
+            else:
+                down = download
+                up = upload
             log.debug(
-                f"Adjust Queue {queue['name']}: max_limit {int(upload)}/{int(download)}")
+                f"Adjust Queue {queue['name']}: max_limit {int(up)}/{int(down)}")
             try:
                 if config["ros_du_invert"] == True:
                     # Inverting upload and download values, because master queue is most likely applied to the bridge
-                    list_queues.set(id=queue['id'], max_limit=f"{int(download)}/{int(upload)}")
+                    list_queues.set(id=queue['id'], max_limit=f"{int(down)}/{int(up)}")
                 else:
                     # Not inverting, use this in case master queue is applied to something like LTE interface
-                    list_queues.set(id=queue['id'], max_limit=f"{int(upload)}/{int(download)}")
+                    list_queues.set(id=queue['id'], max_limit=f"{int(up)}/{int(down)}")
             except:
                 log.error("Unable to change queue settings.", exc_info=True)
     connection.disconnect()
 
 
+data_dict = {}
 wan_upload = None
 wan_download = None
 results_dict = None
@@ -208,6 +244,7 @@ def reset_globals():
     global test_started
     global speedtest_failed
     global downloading
+    global data_dict
 
     wan_upload = None
     wan_download = None
@@ -215,6 +252,7 @@ def reset_globals():
     test_started = False
     speedtest_failed = False
     downloading = True
+    data_dict = {}
 
 
 def threaded_speedtest():
@@ -246,6 +284,7 @@ def threaded_wan_speed():
     global test_started
     global results_dict
     global downloading
+    global data_dict
     print("Waiting for test to start...")
     while not test_started:
         if speedtest_failed:
@@ -253,11 +292,16 @@ def threaded_wan_speed():
         time.sleep(1)
     print("Download warm-up...")
     time.sleep(2)
-    uploads = []
+    uploads = {"total": []}
-    downloads = []
+    downloads = {"total": []}
+
+    for interface in config['ros_wan_interface']:
+        uploads[interface] = []
+        downloads[interface] = []
     print("Monitoring...")
     upload_warmed_up = False
     while results_dict == None:
+        print("\n\n---\n")
         connection = routeros_api.RouterOsApiPool(config['ros_ip'], username=secrets["ros_login"],
                                                   password=secrets["ros_password"], plaintext_login=True)
         api = connection.get_api()
@@ -271,23 +315,43 @@ def threaded_wan_speed():
             rx += int(traffic['rx-bits-per-second'])
             tx += int(traffic['tx-bits-per-second'])
 
+            if downloading:
+                print(f"{interface} DL: {mbits(int(traffic['rx-bits-per-second']))} mbps;")
+                downloads[interface].append(int(traffic['rx-bits-per-second']))
+            elif upload_warmed_up:
+                print(f"{interface} UP: {mbits(int(traffic['tx-bits-per-second']))} mbps;")
+                uploads[interface].append(int(traffic['tx-bits-per-second']))
+
         if downloading:
-            downloads.append(rx)
+            downloads['total'].append(rx)
-            print(f"DL: {mbits(rx)} mbps;")
+
+            print(f"TOTAL DL: {mbits(rx)} mbps;")
         else:
             if upload_warmed_up:
-                uploads.append(tx)
+                uploads['total'].append(tx)
-                print(f"UP: {mbits(tx)} mbps;")
+
+                print(f"TOTAL UP: {mbits(tx)} mbps;")
             else:
                 print("Upload warm-up...")
                 time.sleep(2)
                 upload_warmed_up = True
 
         time.sleep(1)
+
     global wan_download
     global wan_upload
-    wan_download = median(downloads)
+
-    wan_upload = median(uploads)
+    wan_download = median(downloads['total'])
+    wan_upload = median(uploads['total'])
+
+    data_dict['wan_downloads'] = {}
+    data_dict['wan_uploads'] = {}
+
+    for key, value in downloads.items():
+        data_dict['wan_downloads'][key] = median(value)
+    for key, value in uploads.items():
+        data_dict['wan_uploads'][key] = median(value)
+
     log.info(f"Monitor result: {mbits(wan_download)} mbps;  {mbits(wan_upload)} mbps;")
 
 
@@ -327,6 +391,7 @@ def on_fail_or_no_connection():
     entry = Entry()
     entry.upload = None
     entry.download = None
+    entry.data = None
     entry.save()
 
     generate_database_reports()
@@ -343,15 +408,41 @@ def on_fail_or_no_connection():
     log.warning("No internet connection! Exiting.")
     sys.exit()
 
-def range_convert(value, min_old, max_old, min_new, max_new, clamp=True):
+
+def range_convert(value, min_old, max_old, min_new, max_new, clamp=True, integer=True):
     a = (((value - min_old) * (max_new - min_new)) / (max_old - min_old)) + min_new
     if clamp:
         if a > max_new:
             a = max_new
         elif a < min_new:
             a = min_new
+    if integer:
+        a = int(a)
     return a
 
+
+def bits_to_quality(mbps, interface='total', percent=False, integer=True):
+    interface_data = config['ros_quality']
+    if interface in interface_data:
+        d = interface_data[interface]
+        if percent:
+            return range_convert(mbits(mbps), d[0], d[1], 1, 100, integer=integer)
+        else:
+            return range_convert(mbits(mbps), d[0], d[1], 1, 5, integer=integer)
+    else:
+        return 0
+
+
+def make_quality_dict(d):
+    data = {}
+    for key, value in d.items():
+        if key not in data:
+            data[key] = {}
+        data[key]['quality'] = bits_to_quality(value, key)
+        data[key]['quality_percent'] = bits_to_quality(value, key, True)
+    return data
+
+
 if __name__ == "__main__":
     '''
     This script will run a few speed tests, calculate average upload and download speeds and record them into database.
@@ -367,7 +458,9 @@ if __name__ == "__main__":
         log.debug("Test internet connection...")
 
         if config["ros_dynamic_speed"]:
-            ros_fastrack_enable(True)
+            # ros_fastrack_enable(True)
+            # Temporary disable limits by adjusting queues
+            ros_unlimited_speed()  # Thats 4096 Megabits adjust if you got more...
             time.sleep(5)
 
         test_speed()
@@ -378,29 +471,34 @@ if __name__ == "__main__":
         entry = Entry()
         entry.upload = mbits(wan_upload)
         entry.download = mbits(wan_download)
+        entry.data = data_dict
         entry.save()
 
-
-
         if wan_download < config['ros_minimum_speed']:
             wan_download = config['ros_minimum_speed']
 
         if config["ros_dynamic_speed"]:
             ros_dynamic_speed(wan_upload, wan_download)
-            ros_fastrack_enable(False)
+            # ros_fastrack_enable(False)
 
         generate_database_reports()
-
+        data_dict.update({
-        dash_data({
+            "download_quality": make_quality_dict(data_dict['wan_downloads']),
-            "wan_download": mbits(wan_download),
+            "upload_quality": make_quality_dict(data_dict['wan_uploads']),
-            "wan_upload": mbits(wan_upload),
-            "quality": int(range_convert(mbits(wan_download), config['bad_speed_mbps'], config['good_speed_mbps'], 1, 5)),
-            "quality_percent": int(range_convert(mbits(wan_download), config['bad_speed_mbps'], config['good_speed_mbps'], 1, 100)),
             "datetime": datetime.now()
         })
+        dash_data(data_dict)
+
+        # dash_data({
+        #     "wan_download": mbits(wan_download),
+        #     "wan_upload": mbits(wan_upload),
+        #     "quality": int(range_convert(mbits(wan_download), config['bad_speed_mbps'], config['good_speed_mbps'], 1, 5)),
+        #     "quality_percent": int(range_convert(mbits(wan_download), config['bad_speed_mbps'], config['good_speed_mbps'], 1, 100)),
+        #     "datetime": datetime.now()
+        # })
 
     except:
         log.error("Error!", exc_info=True)
 
-        if config["ros_dynamic_speed"]:
+        # if config["ros_dynamic_speed"]:
-            ros_fastrack_enable(False)
+        #     ros_fastrack_enable(False)