From 4767d23db30e4adc09c8173ab71c4c8c63db1512 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 31 Mar 2018 09:08:37 -0500 Subject: [PATCH 1/9] run_tests: factor run_one_test to function --- tests/run-tests | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/run-tests b/tests/run-tests index f1035c4353..3f8a2a8273 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -354,7 +354,9 @@ def run_tests(pyb, tests, args, base_path="."): skip_tests.add('micropython/heapalloc_iter.py') # requires generators skip_tests.add('micropython/schedule.py') # native code doesn't check pending events - for test_file in tests: + def run_one_test(test_file): + nonlocal test_count, testcase_count, passed_count, failed_tests + test_file = test_file.replace('\\', '/') test_basename = os.path.basename(test_file) test_name = os.path.splitext(test_basename)[0] @@ -377,7 +379,7 @@ def run_tests(pyb, tests, args, base_path="."): if skip_it: print("skip ", test_file) skipped_tests.append(test_name) - continue + return # get expected output test_file_expected = test_file + '.exp' @@ -405,7 +407,7 @@ def run_tests(pyb, tests, args, base_path="."): output_expected = output_expected.replace(b'\r\n', b'\n') if args.write_exp: - continue + return # run MicroPython output_mupy = run_micropython(pyb, args, test_file) @@ -413,7 +415,7 @@ def run_tests(pyb, tests, args, base_path="."): if output_mupy == b'SKIP\n': print("skip ", test_file) skipped_tests.append(test_name) - continue + return testcase_count += len(output_expected.splitlines()) @@ -439,6 +441,9 @@ def run_tests(pyb, tests, args, base_path="."): test_count += 1 + for test_file in tests: + run_one_test(test_file) + print("{} tests performed ({} individual testcases)".format(test_count, testcase_count)) print("{} tests passed".format(passed_count)) From a73f005e0030f014400884a8474181fc9d5464ec Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 31 Mar 2018 09:15:29 -0500 Subject: [PATCH 2/9] run_tests: make access to shared variables thread safe --- tests/run-tests | 45 +++++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/tests/run-tests b/tests/run-tests index 3f8a2a8273..4cf16131f5 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -6,6 +6,7 @@ import sys import platform import argparse import re +import threading from glob import glob # Tests require at least CPython 3.3. If your default python3 executable @@ -197,13 +198,27 @@ def run_micropython(pyb, args, test_file, is_special=False): def run_feature_check(pyb, args, base_path, test_file): return run_micropython(pyb, args, base_path + "/feature_check/" + test_file, is_special=True) +class ThreadSafeCounter: + def __init__(self, start=0): + self._value = start + self._lock = threading.Lock() + + def add(self, to_add): + with self._lock: self._value += to_add + + def append(self, arg): + self.add([arg]) + + @property + def value(self): + return self._value def run_tests(pyb, tests, args, base_path="."): - test_count = 0 - testcase_count = 0 - passed_count = 0 - failed_tests = [] - skipped_tests = [] + test_count = ThreadSafeCounter() + testcase_count = ThreadSafeCounter() + passed_count = ThreadSafeCounter() + failed_tests = ThreadSafeCounter([]) + skipped_tests = ThreadSafeCounter([]) skip_tests = set() skip_native = False @@ -355,8 +370,6 @@ def run_tests(pyb, tests, args, base_path="."): skip_tests.add('micropython/schedule.py') # native code doesn't check pending events def run_one_test(test_file): - nonlocal test_count, testcase_count, passed_count, failed_tests - test_file = test_file.replace('\\', '/') test_basename = os.path.basename(test_file) test_name = os.path.splitext(test_basename)[0] @@ -417,14 +430,14 @@ def run_tests(pyb, tests, args, base_path="."): skipped_tests.append(test_name) return - testcase_count += len(output_expected.splitlines()) + testcase_count.add(len(output_expected.splitlines())) filename_expected = test_basename + ".exp" filename_mupy = test_basename + ".out" if output_expected == output_mupy: print("pass ", test_file) - passed_count += 1 + passed_count.add(1) rm_f(filename_expected) rm_f(filename_mupy) else: @@ -439,18 +452,18 @@ def run_tests(pyb, tests, args, base_path="."): print("FAIL ", test_file) failed_tests.append(test_name) - test_count += 1 + test_count.add(1) for test_file in tests: run_one_test(test_file) - print("{} tests performed ({} individual testcases)".format(test_count, testcase_count)) - print("{} tests passed".format(passed_count)) + print("{} tests performed ({} individual testcases)".format(test_count.value, testcase_count.value)) + print("{} tests passed".format(passed_count.value)) - if len(skipped_tests) > 0: - print("{} tests skipped: {}".format(len(skipped_tests), ' '.join(skipped_tests))) - if len(failed_tests) > 0: - print("{} tests failed: {}".format(len(failed_tests), ' '.join(failed_tests))) + if len(skipped_tests.value) > 0: + print("{} tests skipped: {}".format(len(skipped_tests.value), ' '.join(skipped_tests.value))) + if len(failed_tests.value) > 0: + print("{} tests failed: {}".format(len(failed_tests.value), ' '.join(failed_tests.value))) return False # all tests succeeded From b9dd6a5bb4556f848b016dd61b399d07d83a21e0 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 31 Mar 2018 09:34:01 -0500 Subject: [PATCH 3/9] run-tests: sort skipped and failed tests .. otherwise the line which reports tests skipped and failed can come in different orders when -j values above 1 are used. --- tests/run-tests | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/run-tests b/tests/run-tests index 4cf16131f5..f28376ba64 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -461,9 +461,9 @@ def run_tests(pyb, tests, args, base_path="."): print("{} tests passed".format(passed_count.value)) if len(skipped_tests.value) > 0: - print("{} tests skipped: {}".format(len(skipped_tests.value), ' '.join(skipped_tests.value))) + print("{} tests skipped: {}".format(len(skipped_tests.value), ' '.join(sorted(skipped_tests.value)))) if len(failed_tests.value) > 0: - print("{} tests failed: {}".format(len(failed_tests.value), ' '.join(failed_tests.value))) + print("{} tests failed: {}".format(len(failed_tests.value), ' '.join(sorted(failed_tests.value)))) return False # all tests succeeded From a3309ebb809ba8f881d837aaef4997e2adab928a Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 31 Mar 2018 09:32:20 -0500 Subject: [PATCH 4/9] run-tests: optionally parallelize tests When requested via 'run-tests -j', more than one test will be run at a time. On my system, (i5-3320m with 4 threads / 2 cores), this reduces elapsed time by over 50% when testing pots/unix/micropython. Elapsed time, seconds, best of 3 runs with each -j value: before patchset: 18.1 -j1: 18.1 -j2: 11.3 (-37%) -j4: 8.7 (-52%) -j6: 8.4 (-54%) In all cases the final output is identical: 651 tests performed (18932 individual testcases) 651 tests passed 23 tests skipped: buffered_writer... though the individual pass/fail messages can be different/interleaved. --- tests/run-tests | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/run-tests b/tests/run-tests index f28376ba64..59b45d695a 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -7,6 +7,7 @@ import platform import argparse import re import threading +from multiprocessing.pool import ThreadPool from glob import glob # Tests require at least CPython 3.3. If your default python3 executable @@ -213,7 +214,7 @@ class ThreadSafeCounter: def value(self): return self._value -def run_tests(pyb, tests, args, base_path="."): +def run_tests(pyb, tests, args, base_path=".", num_threads=1): test_count = ThreadSafeCounter() testcase_count = ThreadSafeCounter() passed_count = ThreadSafeCounter() @@ -454,8 +455,12 @@ def run_tests(pyb, tests, args, base_path="."): test_count.add(1) - for test_file in tests: - run_one_test(test_file) + if num_threads > 1: + pool = ThreadPool(num_threads) + pool.map(run_one_test, tests) + else: + for test in tests: + run_one_test(test) print("{} tests performed ({} individual testcases)".format(test_count.value, testcase_count.value)) print("{} tests passed".format(passed_count.value)) @@ -482,6 +487,7 @@ def main(): cmd_parser.add_argument('--heapsize', help='heapsize to use (use default if not specified)') cmd_parser.add_argument('--via-mpy', action='store_true', help='compile .py files to .mpy first') cmd_parser.add_argument('--keep-path', action='store_true', help='do not clear MICROPYPATH when running tests') + cmd_parser.add_argument('-j', '--jobs', default=1, metavar='N', type=int, help='Number of tests to run simultaneously') cmd_parser.add_argument('files', nargs='*', help='input test files') args = cmd_parser.parse_args() @@ -525,7 +531,7 @@ def main(): # run-tests script itself. base_path = os.path.dirname(sys.argv[0]) or "." try: - res = run_tests(pyb, tests, args, base_path) + res = run_tests(pyb, tests, args, base_path, args.jobs) finally: if pyb: pyb.close() From c2b85296984d0c65abff44810b1e8b1d99981218 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 31 Mar 2018 09:38:25 -0500 Subject: [PATCH 5/9] run-tests: automatically parallelism based on CPU (thread) count --- tests/run-tests | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/run-tests b/tests/run-tests index 59b45d695a..1e4001bc18 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -7,6 +7,7 @@ import platform import argparse import re import threading +import multiprocessing from multiprocessing.pool import ThreadPool from glob import glob @@ -488,6 +489,7 @@ def main(): cmd_parser.add_argument('--via-mpy', action='store_true', help='compile .py files to .mpy first') cmd_parser.add_argument('--keep-path', action='store_true', help='do not clear MICROPYPATH when running tests') cmd_parser.add_argument('-j', '--jobs', default=1, metavar='N', type=int, help='Number of tests to run simultaneously') + cmd_parser.add_argument('--auto-jobs', action='store_const', dest='jobs', const=multiprocessing.cpu_count(), help='Set the -j values to the CPU (thread) count') cmd_parser.add_argument('files', nargs='*', help='input test files') args = cmd_parser.parse_args() From c1cd259529028bef5048e589c763c2417072db5b Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 31 Mar 2018 09:57:51 -0500 Subject: [PATCH 6/9] travis.yml: best guesses about when to run tests in parallel Notably, "--via-mpy" spews failures when threaded, possibly indicating that micropython is not creating mpy files in threadsafe manner. --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index efb4ec35ff..bd906cff33 100644 --- a/.travis.yml +++ b/.travis.yml @@ -78,24 +78,24 @@ script: - echo -en 'travis_fold:end:qemu\\r' # run tests without coverage info - #- (cd tests && MICROPY_CPYTHON3=python3.4 ./run-tests) - #- (cd tests && MICROPY_CPYTHON3=python3.4 ./run-tests --emit native) + #- (cd tests && MICROPY_CPYTHON3=python3.4 ./run-tests --auto-jobs) + #- (cd tests && MICROPY_CPYTHON3=python3.4 ./run-tests --auto-jobs --emit native) # run tests with coverage info - echo 'Test all' && echo -en 'travis_fold:start:test_all\\r' - - ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests)) + - ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --auto-jobs)) - echo -en 'travis_fold:end:test_all\\r' - echo 'Test threads' && echo -en 'travis_fold:start:test_threads\\r' - - ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests -d thread)) + - ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --auto-jobs -d thread)) - echo -en 'travis_fold:end:test_threads\\r' - echo 'Testing with native' && echo -en 'travis_fold:start:test_native\\r' - - ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --emit native)) + - ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --auto-jobs --emit native)) - echo -en 'travis_fold:end:test_native\\r' - (echo 'Testing with mpy' && echo -en 'travis_fold:start:test_mpy\\r') - - ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --via-mpy -d basics float)) + - ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests -j1 --via-mpy -d basics float)) - echo -en 'travis_fold:end:test_mpy\\r' - (echo 'Building docs' && echo -en 'travis_fold:start:build_docs\\r') From 0dfc3be9033055ead0a27bbd46524de946454076 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 31 Mar 2018 10:00:32 -0500 Subject: [PATCH 7/9] run_tests: EXTERNAL_TARGETS can't run in parallel --- tests/run-tests | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/run-tests b/tests/run-tests index 1e4001bc18..4f05a4af61 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -495,6 +495,7 @@ def main(): EXTERNAL_TARGETS = ('pyboard', 'wipy', 'esp8266', 'minimal') if args.target in EXTERNAL_TARGETS: + args.jobs = 1 import pyboard pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password) pyb.enter_raw_repl() From b59964f70723d29259e1656bed56bb64e2231cee Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 31 Mar 2018 10:03:32 -0500 Subject: [PATCH 8/9] ports/unix/Makefile: parallelize tests --- ports/unix/Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ports/unix/Makefile b/ports/unix/Makefile index 5bdcf4e1a5..dacf680195 100644 --- a/ports/unix/Makefile +++ b/ports/unix/Makefile @@ -189,7 +189,7 @@ include $(TOP)/py/mkrules.mk test: $(PROG) $(TOP)/tests/run-tests $(eval DIRNAME=ports/$(notdir $(CURDIR))) - cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests + cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests --auto-jobs # install micropython in /usr/local/bin TARGET = micropython @@ -254,10 +254,10 @@ coverage: coverage_test: coverage $(eval DIRNAME=ports/$(notdir $(CURDIR))) - cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests - cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests -d thread - cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests --emit native - cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests --via-mpy -d basics float + cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests --auto-jobs + cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests --auto-jobs -d thread + cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests --auto-jobs --emit native + cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests -j1 --via-mpy -d basics float gcov -o build-coverage/py $(TOP)/py/*.c gcov -o build-coverage/extmod $(TOP)/extmod/*.c From f8e0baa0b7c7d341b4ce4aee2d065b6ea213d90b Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 31 Mar 2018 10:03:41 -0500 Subject: [PATCH 9/9] appveyor: parallelize tests --- ports/windows/.appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ports/windows/.appveyor.yml b/ports/windows/.appveyor.yml index a82cf5adc9..9435ce6e67 100644 --- a/ports/windows/.appveyor.yml +++ b/ports/windows/.appveyor.yml @@ -22,7 +22,7 @@ test_script: - cmd: >- cd tests - %MICROPY_CPYTHON3% run-tests + %MICROPY_CPYTHON3% run-tests --auto-jobs skip_tags: true