Python 3.10 support

Fixes #628

.travis.yml

@@ -1,12 +1,14 @@
 language: python
+sudo: required
+dist: xenial
 
 addons:
   apt:
     sources:
       - deadsnakes
     packages:
-      # - python2.4
+      - python2.4
-      # - python2.5
+      - python2.5
       - python2.6
       - python3.2
       - python3.3
@@ -15,10 +17,10 @@ matrix:
   include:
     - python: 2.7
       env: TOXENV=flake8
-    # - python: 2.7
+    - python: 2.7
-    #   env: TOXENV=py24
+      env: TOXENV=py24
-    # - python: 2.7
+    - python: 2.7
-    #   env: TOXENV=py25
+      env: TOXENV=py25
     - python: 2.7
       env: TOXENV=py26
     - python: 2.7
@@ -33,8 +35,13 @@ matrix:
       env: TOXENV=py35
     - python: 3.6
       env: TOXENV=py36
+    - python: 3.7
+      env: TOXENV=py37
+    - python: 3.8-dev
+      env: TOXENV=py38
     - python: pypy
       env: TOXENV=pypy
+      dist: trusty
 
 before_install:
   - if [[ $(echo "$TOXENV" | egrep -c "py35") != 0 ]]; then pyenv global system 3.5; fi;

setup.py

@@ -92,5 +92,8 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
     ]
 )

speedtest.py

@@ -15,18 +15,18 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import os
-import re
 import csv
-import sys
+import datetime
-import math
 import errno
+import math
+import os
+import platform
+import re
 import signal
 import socket
-import timeit
+import sys
-import datetime
-import platform
 import threading
+import timeit
 import xml.parsers.expat
 
 try:
@@ -36,7 +36,7 @@ except ImportError:
     gzip = None
     GZIP_BASE = object
 
-__version__ = '2.1.0'
+__version__ = '2.1.4b1'
 
 
 class FakeShutdownEvent(object):
@@ -49,10 +49,16 @@ class FakeShutdownEvent(object):
         "Dummy method to always return false"""
         return False
 
+    is_set = isSet
+
 
 # Some global variables we use
 DEBUG = False
 _GLOBAL_DEFAULT_TIMEOUT = object()
+PY25PLUS = sys.version_info[:2] >= (2, 5)
+PY26PLUS = sys.version_info[:2] >= (2, 6)
+PY32PLUS = sys.version_info[:2] >= (3, 2)
+PY310PLUS = sys.version_info[:2] >= (3, 10)
 
 # Begin import game to handle Python 2 and Python 3
 try:
@@ -64,14 +70,15 @@ except ImportError:
         json = None
 
 try:
-    import xml.etree.cElementTree as ET
+    import xml.etree.ElementTree as ET
-except ImportError:
     try:
-        import xml.etree.ElementTree as ET
+        from xml.etree.ElementTree import _Element as ET_Element
     except ImportError:
-        from xml.dom import minidom as DOM
+        pass
-        from xml.parsers.expat import ExpatError
+except ImportError:
-        ET = None
+    from xml.dom import minidom as DOM
+    from xml.parsers.expat import ExpatError
+    ET = None
 
 try:
     from urllib2 import (urlopen, Request, HTTPError, URLError,
@@ -97,6 +104,11 @@ except ImportError:
     except ImportError:
         HTTPSConnection = None
 
+try:
+    from httplib import FakeSocket
+except ImportError:
+    FakeSocket = None
+
 try:
     from Queue import Queue
 except ImportError:
@@ -257,7 +269,6 @@ else:
             write(arg)
         write(end)
 
-
 # Exception "constants" to support Python 2 through Python 3
 try:
     import ssl
@@ -274,6 +285,23 @@ except ImportError:
     ssl = None
     HTTP_ERRORS = (HTTPError, URLError, socket.error, BadStatusLine)
 
+if PY32PLUS:
+    etree_iter = ET.Element.iter
+elif PY25PLUS:
+    etree_iter = ET_Element.getiterator
+
+if PY26PLUS:
+    thread_is_alive = threading.Thread.is_alive
+else:
+    thread_is_alive = threading.Thread.isAlive
+
+
+def event_is_set(event):
+    try:
+        return event.is_set()
+    except AttributeError:
+        return event.isSet()
+
 
 class SpeedtestException(Exception):
     """Base exception for this module"""
@@ -394,6 +422,8 @@ class SpeedtestHTTPConnection(HTTPConnection):
         source_address = kwargs.pop('source_address', None)
         timeout = kwargs.pop('timeout', 10)
 
+        self._tunnel_host = None
+
         HTTPConnection.__init__(self, *args, **kwargs)
 
         self.source_address = source_address
@@ -414,17 +444,23 @@ class SpeedtestHTTPConnection(HTTPConnection):
                 self.source_address
             )
 
+        if self._tunnel_host:
+            self._tunnel()
+
 
 if HTTPSConnection:
-    class SpeedtestHTTPSConnection(HTTPSConnection,
+    class SpeedtestHTTPSConnection(HTTPSConnection):
-                                   SpeedtestHTTPConnection):
         """Custom HTTPSConnection to support source_address across
         Python 2.4 - Python 3
         """
+        default_port = 443
+
         def __init__(self, *args, **kwargs):
             source_address = kwargs.pop('source_address', None)
             timeout = kwargs.pop('timeout', 10)
 
+            self._tunnel_host = None
+
             HTTPSConnection.__init__(self, *args, **kwargs)
 
             self.timeout = timeout
@@ -432,14 +468,30 @@ if HTTPSConnection:
 
         def connect(self):
             "Connect to a host on a given (SSL) port."
+            try:
+                self.sock = socket.create_connection(
+                    (self.host, self.port),
+                    self.timeout,
+                    self.source_address
+                )
+            except (AttributeError, TypeError):
+                self.sock = create_connection(
+                    (self.host, self.port),
+                    self.timeout,
+                    self.source_address
+                )
 
-            SpeedtestHTTPConnection.connect(self)
+            if self._tunnel_host:
+                self._tunnel()
 
             if ssl:
                 try:
                     kwargs = {}
                     if hasattr(ssl, 'SSLContext'):
-                        kwargs['server_hostname'] = self.host
+                        if self._tunnel_host:
+                            kwargs['server_hostname'] = self._tunnel_host
+                        else:
+                            kwargs['server_hostname'] = self.host
                     self.sock = self._context.wrap_socket(self.sock, **kwargs)
                 except AttributeError:
                     self.sock = ssl.wrap_socket(self.sock)
@@ -447,6 +499,20 @@ if HTTPSConnection:
                         self.sock.server_hostname = self.host
                     except AttributeError:
                         pass
+            elif FakeSocket:
+                # Python 2.4/2.5 support
+                try:
+                    self.sock = FakeSocket(self.sock, socket.ssl(self.sock))
+                except AttributeError:
+                    raise SpeedtestException(
+                        'This version of Python does not support HTTPS/SSL '
+                        'functionality'
+                    )
+            else:
+                raise SpeedtestException(
+                    'This version of Python does not support HTTPS/SSL '
+                    'functionality'
+                )
 
 
 def _build_connection(connection, source_address, timeout, context=None):
@@ -712,7 +778,7 @@ def print_dots(shutdown_event):
     status
     """
     def inner(current, total, start=False, end=False):
-        if shutdown_event.isSet():
+        if event_is_set(shutdown_event):
             return
 
         sys.stdout.write('.')
@@ -751,7 +817,7 @@ class HTTPDownloader(threading.Thread):
         try:
             if (timeit.default_timer() - self.starttime) <= self.timeout:
                 f = self._opener(self.request)
-                while (not self._shutdown_event.isSet() and
+                while (not event_is_set(self._shutdown_event) and
                         (timeit.default_timer() - self.starttime) <=
                         self.timeout):
                     self.result.append(len(f.read(10240)))
@@ -760,6 +826,8 @@ class HTTPDownloader(threading.Thread):
                 f.close()
         except IOError:
             pass
+        except HTTP_ERRORS:
+            pass
 
 
 class HTTPUploaderData(object):
@@ -805,7 +873,7 @@ class HTTPUploaderData(object):
 
     def read(self, n=10240):
         if ((timeit.default_timer() - self.start) <= self.timeout and
-                not self._shutdown_event.isSet()):
+                not event_is_set(self._shutdown_event)):
             chunk = self.data.read(n)
             self.total.append(len(chunk))
             return chunk
@@ -825,7 +893,7 @@ class HTTPUploader(threading.Thread):
         self.request = request
         self.request.data.start = self.starttime = start
         self.size = size
-        self.result = None
+        self.result = 0
         self.timeout = timeout
         self.i = i
 
@@ -843,7 +911,7 @@ class HTTPUploader(threading.Thread):
         request = self.request
         try:
             if ((timeit.default_timer() - self.starttime) <= self.timeout and
-                    not self._shutdown_event.isSet()):
+                    not event_is_set(self._shutdown_event)):
                 try:
                     f = self._opener(request)
                 except TypeError:
@@ -860,6 +928,8 @@ class HTTPUploader(threading.Thread):
                 self.result = 0
         except (IOError, SpeedtestUploadTimeout):
             self.result = sum(self.request.data.total)
+        except HTTP_ERRORS:
+            self.result = 0
 
 
 class SpeedtestResults(object):
@@ -1113,9 +1183,9 @@ class Speedtest(object):
             # times = get_attributes_by_tag_name(root, 'times')
             client = get_attributes_by_tag_name(root, 'client')
 
-        ignore_servers = list(
+        ignore_servers = [
-            map(int, server_config['ignoreids'].split(','))
+            int(i) for i in server_config['ignoreids'].split(',') if i
-        )
+        ]
 
         ratio = int(upload['ratio'])
         upload_max = int(upload['maxchunkcount'])
@@ -1243,7 +1313,7 @@ class Speedtest(object):
                             raise SpeedtestServersError(
                                 'Malformed speedtest.net server list: %s' % e
                             )
-                        elements = root.getiterator('server')
+                        elements = etree_iter(root, 'server')
                     except AttributeError:
                         try:
                             root = DOM.parseString(serversxml)
@@ -1463,6 +1533,9 @@ class Speedtest(object):
                 build_request(url, bump=i, secure=self._secure)
             )
 
+        max_threads = threads or self.config['threads']['download']
+        in_flight = {'threads': 0}
+
         def producer(q, requests, request_count):
             for i, request in enumerate(requests):
                 thread = HTTPDownloader(
@@ -1473,21 +1546,26 @@ class Speedtest(object):
                     opener=self._opener,
                     shutdown_event=self._shutdown_event
                 )
+                while in_flight['threads'] >= max_threads:
+                    timeit.time.sleep(0.001)
                 thread.start()
                 q.put(thread, True)
+                in_flight['threads'] += 1
                 callback(i, request_count, start=True)
 
         finished = []
 
         def consumer(q, request_count):
+            _is_alive = thread_is_alive
             while len(finished) < request_count:
                 thread = q.get(True)
-                while thread.isAlive():
+                while _is_alive(thread):
-                    thread.join(timeout=0.1)
+                    thread.join(timeout=0.001)
+                in_flight['threads'] -= 1
                 finished.append(sum(thread.result))
                 callback(thread.i, request_count, end=True)
 
-        q = Queue(threads or self.config['threads']['download'])
+        q = Queue(max_threads)
         prod_thread = threading.Thread(target=producer,
                                        args=(q, requests, request_count))
         cons_thread = threading.Thread(target=consumer,
@@ -1495,10 +1573,11 @@ class Speedtest(object):
         start = timeit.default_timer()
         prod_thread.start()
         cons_thread.start()
-        while prod_thread.isAlive():
+        _is_alive = thread_is_alive
-            prod_thread.join(timeout=0.1)
+        while _is_alive(prod_thread):
-        while cons_thread.isAlive():
+            prod_thread.join(timeout=0.001)
-            cons_thread.join(timeout=0.1)
+        while _is_alive(cons_thread):
+            cons_thread.join(timeout=0.001)
 
         stop = timeit.default_timer()
         self.results.bytes_received = sum(finished)
@@ -1547,6 +1626,9 @@ class Speedtest(object):
                 )
             )
 
+        max_threads = threads or self.config['threads']['upload']
+        in_flight = {'threads': 0}
+
         def producer(q, requests, request_count):
             for i, request in enumerate(requests[:request_count]):
                 thread = HTTPUploader(
@@ -1558,17 +1640,22 @@ class Speedtest(object):
                     opener=self._opener,
                     shutdown_event=self._shutdown_event
                 )
+                while in_flight['threads'] >= max_threads:
+                    timeit.time.sleep(0.001)
                 thread.start()
                 q.put(thread, True)
+                in_flight['threads'] += 1
                 callback(i, request_count, start=True)
 
         finished = []
 
         def consumer(q, request_count):
+            _is_alive = thread_is_alive
             while len(finished) < request_count:
                 thread = q.get(True)
-                while thread.isAlive():
+                while _is_alive(thread):
-                    thread.join(timeout=0.1)
+                    thread.join(timeout=0.001)
+                in_flight['threads'] -= 1
                 finished.append(thread.result)
                 callback(thread.i, request_count, end=True)
 
@@ -1580,9 +1667,10 @@ class Speedtest(object):
         start = timeit.default_timer()
         prod_thread.start()
         cons_thread.start()
-        while prod_thread.isAlive():
+        _is_alive = thread_is_alive
+        while _is_alive(prod_thread):
             prod_thread.join(timeout=0.1)
-        while cons_thread.isAlive():
+        while _is_alive(cons_thread):
             cons_thread.join(timeout=0.1)
 
         stop = timeit.default_timer()