From 2368085125ab9667d858443e5592ed2e4e26aef7 Mon Sep 17 00:00:00 2001 From: Nixellion Date: Mon, 13 Jul 2020 16:56:10 +0300 Subject: [PATCH] Update for 2 LTEs --- config/config.yaml | 48 ++++++++------ dbo.py | 11 +-- speedtester.py | 162 ++++++++++++++++++++++++++++++++++++--------- 3 files changed, 163 insertions(+), 58 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index 45ad275..45f916a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,27 +1,31 @@ -database_migrate: False -output_image_path: /var/www/downloads/speedgraph_{}.png -output_txt_path: /var/www/downloads/speeds.txt -ros_dynamic_speed: True -ros_ip: 192.168.88.1 -ros_queues: - - all -ros_queues_except: - - dummy -ros_du_invert: False -ros_fasttrack_comment: "defconf: fasttrack" -# Minimum speed which not to cross, if speed is lower than this it will set this speed -ros_minimum_speed: 100000 -# Speed at which to leave fasttrack on. Use if your ROS device's CPU can't handle max speed you get, for example LHG LTE -# starts to show high CPU usage at close to 20mbit. -# Not implemented yet -ros_maximum_speed: 15000000 # 15 mbit -ros_wan_interface: - - eth_LTE6_1 - - eth_LTE_0 api_host: 0.0.0.0 api_port: 1357 bad_speed_mbps: 0.2 +database_migrate: true good_speed_mbps: 8 +output_image_path: /var/www/downloads/speedgraph_{}.png +output_txt_path: /var/www/downloads/speeds.txt +ros_du_invert: True +ros_dynamic_speed: true +ros_fasttrack_comment: 'defconf: fasttrack' +ros_ip: 192.168.88.1 +ros_maximum_speed: 15000000 +ros_minimum_speed: 100000 +ros_queues: + - queue1 +ros_queues_except: + - dummy +ros_wan_interface: + - eth_LTE6_1 + - eth_LTE_0 +ros_quality: + total: + - 1 + - 20 + eth_LTE6_1: + - 1 + - 10 + eth_LTE_0: + - 1 + - 10 - -# Conversion rate 1 mbit : 1000000 bits \ No newline at end of file diff --git a/dbo.py b/dbo.py index eadf8b6..c14dd99 100644 --- a/dbo.py +++ b/dbo.py @@ -45,6 +45,7 @@ class BroModel(Model): class Entry(BroModel): upload = FloatField(null=True) download = FloatField(null=True) + data = TextField(null=True) class Meta: @@ -72,10 +73,10 @@ if config['database_migrate']: migrator = SqliteMigrator(db) - open_count = IntegerField(default=0) + data = TextField(null=True) migrate( - migrator.add_column('Entry', 'open_count', open_count) + migrator.add_column('Entry', 'data', data) ) log.debug("Migration success") log.debug("=====================") @@ -85,13 +86,15 @@ if config['database_migrate']: except: log.error("Could not migrate", exc_info=True) log.debug("=====================") +else: + db.connect() + db.create_tables([Entry]) # endregion log.info(" ".join(["Using DB", str(db), "At path:", str(db_path)])) # On init make sure we create database -db.connect() -db.create_tables([Entry]) + # endregion diff --git a/speedtester.py b/speedtester.py index 1d092af..577e04b 100644 --- a/speedtester.py +++ b/speedtester.py @@ -13,7 +13,6 @@ import speedtest from dbo import Entry import routeros_api - config = read_config() secrets = read_config('secrets') @@ -36,14 +35,16 @@ def gather_data(): dates.append(entry.date_created) return dates, downloads, uploads + + import matplotlib import matplotlib.pyplot as plt + @catch_errors def generate_plot_image(dates, downloads, uploads, name="speed", description="Speed Graph"): log.debug("Genering image output for {}...".format(name)) - dates = matplotlib.dates.date2num(dates) fig = plt.figure(figsize=(15, 3)) plt.plot_date(dates, downloads, linestyle='solid', color="red", linewidth=2, marker=None) @@ -59,7 +60,7 @@ def generate_diff(dates, downloads, uploads): up = [] for i, date in enumerate(dates): curr = downloads[i] if downloads[i] else 0 - prev = downloads[i-1] if downloads[i-1] and i > 0 else 0 + prev = downloads[i - 1] if downloads[i - 1] and i > 0 else 0 dl.append(curr - prev) curr = uploads[i] if uploads[i] else 0 prev = uploads[i - 1] if uploads[i - 1] and i > 0 else 0 @@ -75,6 +76,7 @@ def generate_updown_plot_simple(downloads, uploads, name, description): plt.tight_layout() plt.savefig(read_config()['output_image_path'].format(name)) + @catch_errors def gather_day_median_data(dates, downloads, uploads): log.debug("Gather day median data...") @@ -98,12 +100,14 @@ def gather_day_median_data(dates, downloads, uploads): return dl, up + @catch_errors def generate_day_median(dates, downloads, uploads): dl, up = gather_day_median_data(dates, downloads, uploads) - + generate_updown_plot_simple(dl, up, "day_median", "Day average") + @catch_errors def generate_day_median_diff(dates, downloads, uploads): dl, up = gather_day_median_data(dates, downloads, uploads) @@ -113,8 +117,8 @@ def generate_day_median_diff(dates, downloads, uploads): for i, down in enumerate(dl): if i > 0: - downs.append(dl[i]-dl[i-1]) - ups.append(up[i]-up[i-1]) + downs.append(dl[i] - dl[i - 1]) + ups.append(up[i] - up[i - 1]) else: downs.append(dl[i]) ups.append(up[i]) @@ -161,6 +165,30 @@ def ros_fastrack_enable(enable): connection.disconnect() +@catch_errors +def ros_unlimited_speed(): + connection = routeros_api.RouterOsApiPool(config['ros_ip'], username=secrets["ros_login"], + password=secrets["ros_password"], plaintext_login=True) + api = connection.get_api() + + list_queues = api.get_resource('/queue/simple') + up = down = 4294967295 + for queue in list_queues.get(): + if queue['name'] in config['ros_queues'] or ( + "all" in config['ros_queues'] and queue["name"] not in config['ros_queues_except']): + log.debug(f"Adjust Queue {queue['name']}: max_limit {int(up)}/{int(down)}") + try: + if config["ros_du_invert"] == True: + # Inverting upload and download values, because master queue is most likely applied to the bridge + list_queues.set(id=queue['id'], max_limit=f"{int(down)}/{int(up)}") + else: + # Not inverting, use this in case master queue is applied to something like LTE interface + list_queues.set(id=queue['id'], max_limit=f"{int(up)}/{int(down)}") + except: + log.error("Unable to change queue settings.", exc_info=True) + connection.disconnect() + + @catch_errors def ros_dynamic_speed(upload, download): ''' @@ -168,7 +196,7 @@ def ros_dynamic_speed(upload, download): :param upload: Upload speed from speedtest :param download: Download speed from speedtest ''' - log.debug(f"Set Dynamic Speed to: DOWN {mbits(download)} mbps; UP {mbits(upload)} mbps") + log.debug(f"Set Speed to: DOWN {mbits(download)} mbps; UP {mbits(upload)} mbps") connection = routeros_api.RouterOsApiPool(config['ros_ip'], username=secrets["ros_login"], password=secrets["ros_password"], plaintext_login=True) api = connection.get_api() @@ -176,21 +204,29 @@ def ros_dynamic_speed(upload, download): list_queues = api.get_resource('/queue/simple') for queue in list_queues.get(): - if queue['name'] in config['ros_queues'] or ("all" in config['ros_queues'] and queue["name"] not in config['ros_queues_except']): + if queue['name'] in config['ros_queues'] or ( + "all" in config['ros_queues'] and queue["name"] not in config['ros_queues_except']): + if queue['name'] in data_dict['wan_downloads']: + down = data_dict['wan_downloads'][queue['name']] + up = data_dict['wan_uploads'][queue['name']] + else: + down = download + up = upload log.debug( - f"Adjust Queue {queue['name']}: max_limit {int(upload)}/{int(download)}") + f"Adjust Queue {queue['name']}: max_limit {int(up)}/{int(down)}") try: if config["ros_du_invert"] == True: # Inverting upload and download values, because master queue is most likely applied to the bridge - list_queues.set(id=queue['id'], max_limit=f"{int(download)}/{int(upload)}") + list_queues.set(id=queue['id'], max_limit=f"{int(down)}/{int(up)}") else: # Not inverting, use this in case master queue is applied to something like LTE interface - list_queues.set(id=queue['id'], max_limit=f"{int(upload)}/{int(download)}") + list_queues.set(id=queue['id'], max_limit=f"{int(up)}/{int(down)}") except: log.error("Unable to change queue settings.", exc_info=True) connection.disconnect() +data_dict = {} wan_upload = None wan_download = None results_dict = None @@ -208,6 +244,7 @@ def reset_globals(): global test_started global speedtest_failed global downloading + global data_dict wan_upload = None wan_download = None @@ -215,6 +252,7 @@ def reset_globals(): test_started = False speedtest_failed = False downloading = True + data_dict = {} def threaded_speedtest(): @@ -246,6 +284,7 @@ def threaded_wan_speed(): global test_started global results_dict global downloading + global data_dict print("Waiting for test to start...") while not test_started: if speedtest_failed: @@ -253,11 +292,16 @@ def threaded_wan_speed(): time.sleep(1) print("Download warm-up...") time.sleep(2) - uploads = [] - downloads = [] + uploads = {"total": []} + downloads = {"total": []} + + for interface in config['ros_wan_interface']: + uploads[interface] = [] + downloads[interface] = [] print("Monitoring...") upload_warmed_up = False while results_dict == None: + print("\n\n---\n") connection = routeros_api.RouterOsApiPool(config['ros_ip'], username=secrets["ros_login"], password=secrets["ros_password"], plaintext_login=True) api = connection.get_api() @@ -271,23 +315,43 @@ def threaded_wan_speed(): rx += int(traffic['rx-bits-per-second']) tx += int(traffic['tx-bits-per-second']) + if downloading: + print(f"{interface} DL: {mbits(int(traffic['rx-bits-per-second']))} mbps;") + downloads[interface].append(int(traffic['rx-bits-per-second'])) + elif upload_warmed_up: + print(f"{interface} UP: {mbits(int(traffic['tx-bits-per-second']))} mbps;") + uploads[interface].append(int(traffic['tx-bits-per-second'])) + if downloading: - downloads.append(rx) - print(f"DL: {mbits(rx)} mbps;") + downloads['total'].append(rx) + + print(f"TOTAL DL: {mbits(rx)} mbps;") else: if upload_warmed_up: - uploads.append(tx) - print(f"UP: {mbits(tx)} mbps;") + uploads['total'].append(tx) + + print(f"TOTAL UP: {mbits(tx)} mbps;") else: print("Upload warm-up...") time.sleep(2) upload_warmed_up = True time.sleep(1) + global wan_download global wan_upload - wan_download = median(downloads) - wan_upload = median(uploads) + + wan_download = median(downloads['total']) + wan_upload = median(uploads['total']) + + data_dict['wan_downloads'] = {} + data_dict['wan_uploads'] = {} + + for key, value in downloads.items(): + data_dict['wan_downloads'][key] = median(value) + for key, value in uploads.items(): + data_dict['wan_uploads'][key] = median(value) + log.info(f"Monitor result: {mbits(wan_download)} mbps; {mbits(wan_upload)} mbps;") @@ -327,6 +391,7 @@ def on_fail_or_no_connection(): entry = Entry() entry.upload = None entry.download = None + entry.data = None entry.save() generate_database_reports() @@ -343,15 +408,41 @@ def on_fail_or_no_connection(): log.warning("No internet connection! Exiting.") sys.exit() -def range_convert(value, min_old, max_old, min_new, max_new, clamp=True): + +def range_convert(value, min_old, max_old, min_new, max_new, clamp=True, integer=True): a = (((value - min_old) * (max_new - min_new)) / (max_old - min_old)) + min_new if clamp: if a > max_new: a = max_new elif a < min_new: a = min_new + if integer: + a = int(a) return a + +def bits_to_quality(mbps, interface='total', percent=False, integer=True): + interface_data = config['ros_quality'] + if interface in interface_data: + d = interface_data[interface] + if percent: + return range_convert(mbits(mbps), d[0], d[1], 1, 100, integer=integer) + else: + return range_convert(mbits(mbps), d[0], d[1], 1, 5, integer=integer) + else: + return 0 + + +def make_quality_dict(d): + data = {} + for key, value in d.items(): + if key not in data: + data[key] = {} + data[key]['quality'] = bits_to_quality(value, key) + data[key]['quality_percent'] = bits_to_quality(value, key, True) + return data + + if __name__ == "__main__": ''' This script will run a few speed tests, calculate average upload and download speeds and record them into database. @@ -367,7 +458,9 @@ if __name__ == "__main__": log.debug("Test internet connection...") if config["ros_dynamic_speed"]: - ros_fastrack_enable(True) + # ros_fastrack_enable(True) + # Temporary disable limits by adjusting queues + ros_unlimited_speed() # Thats 4096 Megabits adjust if you got more... time.sleep(5) test_speed() @@ -378,29 +471,34 @@ if __name__ == "__main__": entry = Entry() entry.upload = mbits(wan_upload) entry.download = mbits(wan_download) + entry.data = data_dict entry.save() - - if wan_download < config['ros_minimum_speed']: wan_download = config['ros_minimum_speed'] if config["ros_dynamic_speed"]: ros_dynamic_speed(wan_upload, wan_download) - ros_fastrack_enable(False) + # ros_fastrack_enable(False) generate_database_reports() - - dash_data({ - "wan_download": mbits(wan_download), - "wan_upload": mbits(wan_upload), - "quality": int(range_convert(mbits(wan_download), config['bad_speed_mbps'], config['good_speed_mbps'], 1, 5)), - "quality_percent": int(range_convert(mbits(wan_download), config['bad_speed_mbps'], config['good_speed_mbps'], 1, 100)), + data_dict.update({ + "download_quality": make_quality_dict(data_dict['wan_downloads']), + "upload_quality": make_quality_dict(data_dict['wan_uploads']), "datetime": datetime.now() }) + dash_data(data_dict) + + # dash_data({ + # "wan_download": mbits(wan_download), + # "wan_upload": mbits(wan_upload), + # "quality": int(range_convert(mbits(wan_download), config['bad_speed_mbps'], config['good_speed_mbps'], 1, 5)), + # "quality_percent": int(range_convert(mbits(wan_download), config['bad_speed_mbps'], config['good_speed_mbps'], 1, 100)), + # "datetime": datetime.now() + # }) except: log.error("Error!", exc_info=True) - if config["ros_dynamic_speed"]: - ros_fastrack_enable(False) + # if config["ros_dynamic_speed"]: + # ros_fastrack_enable(False)