Nix
/
BroSpeedTester
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Update for 2 LTEs

master
Nixellion 2020-07-13 16:56:10 +03:00
parent 6caf1f365e
commit 2368085125
3 changed files with 163 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
config/config.yaml
View File

@ -1,27 +1,31 @@
database_migrate: False
output_image_path: /var/www/downloads/speedgraph_{}.png
output_txt_path: /var/www/downloads/speeds.txt
ros_dynamic_speed: True
ros_ip: 192.168.88.1
ros_queues:
- all
ros_queues_except:
- dummy
ros_du_invert: False
ros_fasttrack_comment: "defconf: fasttrack"
# Minimum speed which not to cross, if speed is lower than this it will set this speed
ros_minimum_speed: 100000
# Speed at which to leave fasttrack on. Use if your ROS device's CPU can't handle max speed you get, for example LHG LTE
# starts to show high CPU usage at close to 20mbit.
# Not implemented yet
ros_maximum_speed: 15000000 # 15 mbit
ros_wan_interface:
- eth_LTE6_1
- eth_LTE_0
api_host: 0.0.0.0
api_port: 1357
bad_speed_mbps: 0.2
database_migrate: true
good_speed_mbps: 8
output_image_path: /var/www/downloads/speedgraph_{}.png
output_txt_path: /var/www/downloads/speeds.txt
ros_du_invert: True
ros_dynamic_speed: true
ros_fasttrack_comment: 'defconf: fasttrack'
ros_ip: 192.168.88.1
ros_maximum_speed: 15000000
ros_minimum_speed: 100000
ros_queues:
- queue1
ros_queues_except:
- dummy
ros_wan_interface:
- eth_LTE6_1
- eth_LTE_0
ros_quality:
total:
- 1
- 20
eth_LTE6_1:
- 1
- 10
eth_LTE_0:
- 1
- 10
# Conversion rate 1 mbit : 1000000 bits

11
dbo.py
View File

@ -45,6 +45,7 @@ class BroModel(Model):
class Entry(BroModel):
upload = FloatField(null=True)
download = FloatField(null=True)
data = TextField(null=True)
class Meta:
@ -72,10 +73,10 @@ if config['database_migrate']:
migrator = SqliteMigrator(db)
open_count = IntegerField(default=0)
data = TextField(null=True)
migrate(
migrator.add_column('Entry', 'open_count', open_count)
migrator.add_column('Entry', 'data', data)
)
log.debug("Migration success")
log.debug("=====================")
@ -85,13 +86,15 @@ if config['database_migrate']:
except:
log.error("Could not migrate", exc_info=True)
log.debug("=====================")
else:
db.connect()
db.create_tables([Entry])
# endregion
log.info(" ".join(["Using DB", str(db), "At path:", str(db_path)]))
# On init make sure we create database
db.connect()
db.create_tables([Entry])
# endregion

162
speedtester.py
View File

@ -13,7 +13,6 @@ import speedtest
from dbo import Entry
import routeros_api
config = read_config()
secrets = read_config('secrets')
@ -36,14 +35,16 @@ def gather_data():
dates.append(entry.date_created)
return dates, downloads, uploads
import matplotlib
import matplotlib.pyplot as plt
@catch_errors
def generate_plot_image(dates, downloads, uploads, name="speed", description="Speed Graph"):
log.debug("Genering image output for {}...".format(name))
dates = matplotlib.dates.date2num(dates)
fig = plt.figure(figsize=(15, 3))
plt.plot_date(dates, downloads, linestyle='solid', color="red", linewidth=2, marker=None)
@ -59,7 +60,7 @@ def generate_diff(dates, downloads, uploads):
up = []
for i, date in enumerate(dates):
curr = downloads[i] if downloads[i] else 0
prev = downloads[i-1] if downloads[i-1] and i > 0 else 0
prev = downloads[i - 1] if downloads[i - 1] and i > 0 else 0
dl.append(curr - prev)
curr = uploads[i] if uploads[i] else 0
prev = uploads[i - 1] if uploads[i - 1] and i > 0 else 0
@ -75,6 +76,7 @@ def generate_updown_plot_simple(downloads, uploads, name, description):
plt.tight_layout()
plt.savefig(read_config()['output_image_path'].format(name))
@catch_errors
def gather_day_median_data(dates, downloads, uploads):
log.debug("Gather day median data...")
@ -98,12 +100,14 @@ def gather_day_median_data(dates, downloads, uploads):
return dl, up
@catch_errors
def generate_day_median(dates, downloads, uploads):
dl, up = gather_day_median_data(dates, downloads, uploads)
generate_updown_plot_simple(dl, up, "day_median", "Day average")
@catch_errors
def generate_day_median_diff(dates, downloads, uploads):
dl, up = gather_day_median_data(dates, downloads, uploads)
@ -113,8 +117,8 @@ def generate_day_median_diff(dates, downloads, uploads):
for i, down in enumerate(dl):
if i > 0:
downs.append(dl[i]-dl[i-1])
ups.append(up[i]-up[i-1])
downs.append(dl[i] - dl[i - 1])
ups.append(up[i] - up[i - 1])
else:
downs.append(dl[i])
ups.append(up[i])
@ -161,6 +165,30 @@ def ros_fastrack_enable(enable):
connection.disconnect()
@catch_errors
def ros_unlimited_speed():
connection = routeros_api.RouterOsApiPool(config['ros_ip'], username=secrets["ros_login"],
password=secrets["ros_password"], plaintext_login=True)
api = connection.get_api()
list_queues = api.get_resource('/queue/simple')
up = down = 4294967295
for queue in list_queues.get():
if queue['name'] in config['ros_queues'] or (
"all" in config['ros_queues'] and queue["name"] not in config['ros_queues_except']):
log.debug(f"Adjust Queue {queue['name']}: max_limit {int(up)}/{int(down)}")
try:
if config["ros_du_invert"] == True:
# Inverting upload and download values, because master queue is most likely applied to the bridge
list_queues.set(id=queue['id'], max_limit=f"{int(down)}/{int(up)}")
else:
# Not inverting, use this in case master queue is applied to something like LTE interface
list_queues.set(id=queue['id'], max_limit=f"{int(up)}/{int(down)}")
except:
log.error("Unable to change queue settings.", exc_info=True)
connection.disconnect()
@catch_errors
def ros_dynamic_speed(upload, download):
'''
@ -168,7 +196,7 @@ def ros_dynamic_speed(upload, download):
:param upload: Upload speed from speedtest
:param download: Download speed from speedtest
'''
log.debug(f"Set Dynamic Speed to: DOWN {mbits(download)} mbps; UP {mbits(upload)} mbps")
log.debug(f"Set Speed to: DOWN {mbits(download)} mbps; UP {mbits(upload)} mbps")
connection = routeros_api.RouterOsApiPool(config['ros_ip'], username=secrets["ros_login"],
password=secrets["ros_password"], plaintext_login=True)
api = connection.get_api()
@ -176,21 +204,29 @@ def ros_dynamic_speed(upload, download):
list_queues = api.get_resource('/queue/simple')
for queue in list_queues.get():
if queue['name'] in config['ros_queues'] or ("all" in config['ros_queues'] and queue["name"] not in config['ros_queues_except']):
if queue['name'] in config['ros_queues'] or (
"all" in config['ros_queues'] and queue["name"] not in config['ros_queues_except']):
if queue['name'] in data_dict['wan_downloads']:
down = data_dict['wan_downloads'][queue['name']]
up = data_dict['wan_uploads'][queue['name']]
else:
down = download
up = upload
log.debug(
f"Adjust Queue {queue['name']}: max_limit {int(upload)}/{int(download)}")
f"Adjust Queue {queue['name']}: max_limit {int(up)}/{int(down)}")
try:
if config["ros_du_invert"] == True:
# Inverting upload and download values, because master queue is most likely applied to the bridge
list_queues.set(id=queue['id'], max_limit=f"{int(download)}/{int(upload)}")
list_queues.set(id=queue['id'], max_limit=f"{int(down)}/{int(up)}")
else:
# Not inverting, use this in case master queue is applied to something like LTE interface
list_queues.set(id=queue['id'], max_limit=f"{int(upload)}/{int(download)}")
list_queues.set(id=queue['id'], max_limit=f"{int(up)}/{int(down)}")
except:
log.error("Unable to change queue settings.", exc_info=True)
connection.disconnect()
data_dict = {}
wan_upload = None
wan_download = None
results_dict = None
@ -208,6 +244,7 @@ def reset_globals():
global test_started
global speedtest_failed
global downloading
global data_dict
wan_upload = None
wan_download = None
@ -215,6 +252,7 @@ def reset_globals():
test_started = False
speedtest_failed = False
downloading = True
data_dict = {}
def threaded_speedtest():
@ -246,6 +284,7 @@ def threaded_wan_speed():
global test_started
global results_dict
global downloading
global data_dict
print("Waiting for test to start...")
while not test_started:
if speedtest_failed:
@ -253,11 +292,16 @@ def threaded_wan_speed():
time.sleep(1)
print("Download warm-up...")
time.sleep(2)
uploads = []
downloads = []
uploads = {"total": []}
downloads = {"total": []}
for interface in config['ros_wan_interface']:
uploads[interface] = []
downloads[interface] = []
print("Monitoring...")
upload_warmed_up = False
while results_dict == None:
print("\n\n---\n")
connection = routeros_api.RouterOsApiPool(config['ros_ip'], username=secrets["ros_login"],
password=secrets["ros_password"], plaintext_login=True)
api = connection.get_api()
@ -271,23 +315,43 @@ def threaded_wan_speed():
rx += int(traffic['rx-bits-per-second'])
tx += int(traffic['tx-bits-per-second'])
if downloading:
print(f"{interface} DL: {mbits(int(traffic['rx-bits-per-second']))} mbps;")
downloads[interface].append(int(traffic['rx-bits-per-second']))
elif upload_warmed_up:
print(f"{interface} UP: {mbits(int(traffic['tx-bits-per-second']))} mbps;")
uploads[interface].append(int(traffic['tx-bits-per-second']))
if downloading:
downloads.append(rx)
print(f"DL: {mbits(rx)} mbps;")
downloads['total'].append(rx)
print(f"TOTAL DL: {mbits(rx)} mbps;")
else:
if upload_warmed_up:
uploads.append(tx)
print(f"UP: {mbits(tx)} mbps;")
uploads['total'].append(tx)
print(f"TOTAL UP: {mbits(tx)} mbps;")
else:
print("Upload warm-up...")
time.sleep(2)
upload_warmed_up = True
time.sleep(1)
global wan_download
global wan_upload
wan_download = median(downloads)
wan_upload = median(uploads)
wan_download = median(downloads['total'])
wan_upload = median(uploads['total'])
data_dict['wan_downloads'] = {}
data_dict['wan_uploads'] = {}
for key, value in downloads.items():
data_dict['wan_downloads'][key] = median(value)
for key, value in uploads.items():
data_dict['wan_uploads'][key] = median(value)
log.info(f"Monitor result: {mbits(wan_download)} mbps; {mbits(wan_upload)} mbps;")
@ -327,6 +391,7 @@ def on_fail_or_no_connection():
entry = Entry()
entry.upload = None
entry.download = None
entry.data = None
entry.save()
generate_database_reports()
@ -343,15 +408,41 @@ def on_fail_or_no_connection():
log.warning("No internet connection! Exiting.")
sys.exit()
def range_convert(value, min_old, max_old, min_new, max_new, clamp=True):
def range_convert(value, min_old, max_old, min_new, max_new, clamp=True, integer=True):
a = (((value - min_old) * (max_new - min_new)) / (max_old - min_old)) + min_new
if clamp:
if a > max_new:
a = max_new
elif a < min_new:
a = min_new
if integer:
a = int(a)
return a
def bits_to_quality(mbps, interface='total', percent=False, integer=True):
interface_data = config['ros_quality']
if interface in interface_data:
d = interface_data[interface]
if percent:
return range_convert(mbits(mbps), d[0], d[1], 1, 100, integer=integer)
else:
return range_convert(mbits(mbps), d[0], d[1], 1, 5, integer=integer)
else:
return 0
def make_quality_dict(d):
data = {}
for key, value in d.items():
if key not in data:
data[key] = {}
data[key]['quality'] = bits_to_quality(value, key)
data[key]['quality_percent'] = bits_to_quality(value, key, True)
return data
if __name__ == "__main__":
'''
This script will run a few speed tests, calculate average upload and download speeds and record them into database.
@ -367,7 +458,9 @@ if __name__ == "__main__":
log.debug("Test internet connection...")
if config["ros_dynamic_speed"]:
ros_fastrack_enable(True)
# ros_fastrack_enable(True)
# Temporary disable limits by adjusting queues
ros_unlimited_speed() # Thats 4096 Megabits adjust if you got more...
time.sleep(5)
test_speed()
@ -378,29 +471,34 @@ if __name__ == "__main__":
entry = Entry()
entry.upload = mbits(wan_upload)
entry.download = mbits(wan_download)
entry.data = data_dict
entry.save()
if wan_download < config['ros_minimum_speed']:
wan_download = config['ros_minimum_speed']
if config["ros_dynamic_speed"]:
ros_dynamic_speed(wan_upload, wan_download)
ros_fastrack_enable(False)
# ros_fastrack_enable(False)
generate_database_reports()
dash_data({
"wan_download": mbits(wan_download),
"wan_upload": mbits(wan_upload),
"quality": int(range_convert(mbits(wan_download), config['bad_speed_mbps'], config['good_speed_mbps'], 1, 5)),
"quality_percent": int(range_convert(mbits(wan_download), config['bad_speed_mbps'], config['good_speed_mbps'], 1, 100)),
data_dict.update({
"download_quality": make_quality_dict(data_dict['wan_downloads']),
"upload_quality": make_quality_dict(data_dict['wan_uploads']),
"datetime": datetime.now()
})
dash_data(data_dict)
# dash_data({
# "wan_download": mbits(wan_download),
# "wan_upload": mbits(wan_upload),
# "quality": int(range_convert(mbits(wan_download), config['bad_speed_mbps'], config['good_speed_mbps'], 1, 5)),
# "quality_percent": int(range_convert(mbits(wan_download), config['bad_speed_mbps'], config['good_speed_mbps'], 1, 100)),
# "datetime": datetime.now()
# })
except:
log.error("Error!", exc_info=True)
if config["ros_dynamic_speed"]:
ros_fastrack_enable(False)
# if config["ros_dynamic_speed"]:
# ros_fastrack_enable(False)