[downloader/(common/http)] Changes calculation of the rate-limit. (Fix #2297, fix #2140, fix #595, fix #2370)

This commit is contained in:
SyxbEaEQ2 2014-07-31 03:08:24 +02:00
parent 4f31d0f2b7
commit c7667c2d7f
2 changed files with 31 additions and 15 deletions

View File

@ -77,8 +77,10 @@ class FileDownloader(object):
def calc_eta(start, now, total, current): def calc_eta(start, now, total, current):
if total is None: if total is None:
return None return None
if now is None:
now = time.time()
dif = now - start dif = now - start
if current == 0 or dif < 0.001: # One millisecond if current == 0 or dif < 0.001: # One millisecond
return None return None
rate = float(current) / dif rate = float(current) / dif
return int((float(total) - float(current)) / rate) return int((float(total) - float(current)) / rate)
@ -92,7 +94,7 @@ class FileDownloader(object):
@staticmethod @staticmethod
def calc_speed(start, now, bytes): def calc_speed(start, now, bytes):
dif = now - start dif = now - start
if bytes == 0 or dif < 0.001: # One millisecond if bytes == 0 or dif < 0.001: # One millisecond
return None return None
return float(bytes) / dif return float(bytes) / dif
@ -105,7 +107,7 @@ class FileDownloader(object):
@staticmethod @staticmethod
def best_block_size(elapsed_time, bytes): def best_block_size(elapsed_time, bytes):
new_min = max(bytes / 2.0, 1.0) new_min = max(bytes / 2.0, 1.0)
new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
if elapsed_time < 0.001: if elapsed_time < 0.001:
return int(new_max) return int(new_max)
rate = bytes / elapsed_time rate = bytes / elapsed_time
@ -143,18 +145,19 @@ class FileDownloader(object):
def report_error(self, *args, **kargs): def report_error(self, *args, **kargs):
self.ydl.report_error(*args, **kargs) self.ydl.report_error(*args, **kargs)
def slow_down(self, start_time, byte_counter): def slow_down(self, start_time, now, byte_counter):
"""Sleep if the download speed is over the rate limit.""" """Sleep if the download speed is over the rate limit."""
rate_limit = self.params.get('ratelimit', None) rate_limit = self.params.get('ratelimit', None)
if rate_limit is None or byte_counter == 0: if rate_limit is None or byte_counter == 0:
return return
now = time.time() if now is None:
now = time.time()
elapsed = now - start_time elapsed = now - start_time
if elapsed <= 0.0: if elapsed <= 0.0:
return return
speed = float(byte_counter) / elapsed speed = float(byte_counter) / elapsed
if speed > rate_limit: if speed > rate_limit:
time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit) time.sleep((byte_counter / rate_limit) - elapsed)
def temp_name(self, filename): def temp_name(self, filename):
"""Returns a temporary filename for the given filename.""" """Returns a temporary filename for the given filename."""

View File

@ -128,16 +128,21 @@ class HttpFD(FileDownloader):
byte_counter = 0 + resume_len byte_counter = 0 + resume_len
block_size = self.params.get('buffersize', 1024) block_size = self.params.get('buffersize', 1024)
start = time.time() start = time.time()
# measure time over whole while-loop, so slow_down() and best_block_size() work together properly
now = None # needed for slow_down() in the first loop run
before = start # start measuring
while True: while True:
# Download and write # Download and write
before = time.time()
data_block = data.read(block_size if not is_test else min(block_size, data_len - byte_counter)) data_block = data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
after = time.time()
if len(data_block) == 0:
break
byte_counter += len(data_block) byte_counter += len(data_block)
# Open file just in time # exit loop when download is finished
if len(data_block) == 0:
break
# Open destination file just in time
if stream is None: if stream is None:
try: try:
(stream, tmpfilename) = sanitize_open(tmpfilename, open_mode) (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
@ -153,11 +158,22 @@ class HttpFD(FileDownloader):
self.to_stderr(u"\n") self.to_stderr(u"\n")
self.report_error(u'unable to write data: %s' % str(err)) self.report_error(u'unable to write data: %s' % str(err))
return False return False
# Apply rate limit
self.slow_down(start, now, byte_counter - resume_len)
# end measuring of one loop run
now = time.time()
after = now
# Adjust block size
if not self.params.get('noresizebuffer', False): if not self.params.get('noresizebuffer', False):
block_size = self.best_block_size(after - before, len(data_block)) block_size = self.best_block_size(after - before, len(data_block))
before = after
# Progress message # Progress message
speed = self.calc_speed(start, time.time(), byte_counter - resume_len) speed = self.calc_speed(start, now, byte_counter - resume_len)
if data_len is None: if data_len is None:
eta = percent = None eta = percent = None
else: else:
@ -178,9 +194,6 @@ class HttpFD(FileDownloader):
if is_test and byte_counter == data_len: if is_test and byte_counter == data_len:
break break
# Apply rate limit
self.slow_down(start, byte_counter - resume_len)
if stream is None: if stream is None:
self.to_stderr(u"\n") self.to_stderr(u"\n")
self.report_error(u'Did not get any data blocks') self.report_error(u'Did not get any data blocks')