From 55c69b7b66084f89c06a13da4ccf482ec21d051e Mon Sep 17 00:00:00 2001 From: sommersoft Date: Tue, 7 May 2019 18:12:43 -0500 Subject: [PATCH 1/7] Revert "Revert "Include mpy-cross '__init__.py' In Packages; Restructure Package Bundling"" This reverts commit cd1a858a01527723f1a78c8f7ef742d380809144. --- circuitpython_build_tools/build.py | 47 +++++++++++++++---- .../scripts/build_bundles.py | 14 ++++-- 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index 49244e2..6bedd9e 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -100,31 +100,64 @@ def _munge_to_temp(original_path, temp_file, library_version): temp_file.write(line.encode("utf-8") + b"\r\n") temp_file.flush() -def library(library_path, output_directory, mpy_cross=None, example_bundle=False): +def library(library_path, output_directory, mpy_cross=None, example_bundle=False, pkg_folder_prefix=None): py_files = [] package_files = [] example_files = [] total_size = 512 for filename in os.listdir(library_path): full_path = os.path.join(library_path, filename) - if os.path.isdir(full_path) and filename not in ["docs"]: - files = os.listdir(full_path) - files = filter(lambda x: x.endswith(".py") or x.startswith("font5x8.bin"), files) + if os.path.isdir(full_path): + path_walk = [names for names in os.walk(full_path)] + #print("- '{}' walk: {}".format(filename, path_walk)) + + # iterate through path_walk, appending each file to + # 'walked_files' while retaining subdirectory structure + walked_files = [] + for path in path_walk: + path_tail_idx = path[0].rfind("/") + 1 + path_tail = path[0][path_tail_idx:] + rel_path = "" + # if this entry is the package top dir, keep it + # empty so we don't double append the dir name + if filename not in path_tail: + rel_path = "{}/".format(path_tail) + + for path_files in path[2]: + walked_files.append("{}{}".format(rel_path, path_files)) + #print(" - expanded file walk: {}".format(walked_files)) + + files = filter(lambda x: x.endswith(".py") or x.startswith("font5x8.bin"), walked_files) files = map(lambda x: os.path.join(filename, x), files) + if filename.startswith("examples"): example_files.extend(files) + #print("- example files: {}".format(example_files)) else: + if pkg_folder_prefix: + if (not example_bundle and + not filename.startswith(pkg_folder_prefix)): + #print("skipped path: {}".format(full_path)) + continue if not example_bundle: package_files.extend(files) + #print("- package files: {} | {}".format(filename, package_files)) + if (filename.endswith(".py") and filename not in IGNORE_PY and not example_bundle): - py_files.append(filename) + py_files.append(filename) if len(py_files) > 1: raise ValueError("Multiple top level py files not allowed. Please put them in a package " "or combine them into a single file.") + for fn in example_files: + base_dir = os.path.join(output_directory.replace("/lib", "/"), os.path.dirname(fn)) + if not os.path.isdir(base_dir): + os.makedirs(base_dir) + total_size += 512 + for fn in package_files: base_dir = os.path.join(output_directory, os.path.dirname(fn)) if not os.path.isdir(base_dir): @@ -163,9 +196,7 @@ def library(library_path, output_directory, mpy_cross=None, example_bundle=False full_path = os.path.join(library_path, filename) with tempfile.NamedTemporaryFile() as temp_file: _munge_to_temp(full_path, temp_file, library_version) - if (not mpy_cross or - os.stat(full_path).st_size == 0 or - filename.endswith("__init__.py")): + if not mpy_cross or os.stat(full_path).st_size == 0: output_file = os.path.join(output_directory, filename) shutil.copyfile(temp_file.name, output_file) else: diff --git a/circuitpython_build_tools/scripts/build_bundles.py b/circuitpython_build_tools/scripts/build_bundles.py index 057c1ee..ec9080d 100755 --- a/circuitpython_build_tools/scripts/build_bundles.py +++ b/circuitpython_build_tools/scripts/build_bundles.py @@ -49,7 +49,8 @@ def add_file(bundle, src_file, zip_name): def build_bundle(libs, bundle_version, output_filename, - build_tools_version="devel", mpy_cross=None, example_bundle=False): + build_tools_version="devel", mpy_cross=None, example_bundle=False, + pkg_folder_prefix=None): build_dir = "build-" + os.path.basename(output_filename) top_folder = os.path.basename(output_filename).replace(".zip", "") build_lib_dir = os.path.join(build_dir, top_folder, "lib") @@ -69,8 +70,8 @@ def build_bundle(libs, bundle_version, output_filename, success = True for library_path in libs: try: - build.library(library_path, build_lib_dir, mpy_cross=mpy_cross, - example_bundle=example_bundle) + build.library(library_path, build_lib_dir, pkg_folder_prefix=pkg_folder_prefix, + mpy_cross=mpy_cross, example_bundle=example_bundle) except ValueError as e: print("build.library failure:", library_path) print(e) @@ -135,7 +136,8 @@ def _find_libraries(current_path, depth): @click.option('--output_directory', default="bundles", help="Output location for the zip files.") @click.option('--library_location', required=True, help="Location of libraries to bundle.") @click.option('--library_depth', default=0, help="Depth of library folders. This is useful when multiple libraries are bundled together but are initially in separate subfolders.") -def build_bundles(filename_prefix, output_directory, library_location, library_depth): +@click.option('--package_folder_prefix', default=None, required=False, help="Prefix string used to determine package folders to bundle.") +def build_bundles(filename_prefix, output_directory, library_location, library_depth, pkg_folder_prefix): os.makedirs(output_directory, exist_ok=True) bundle_version = build.version_string() @@ -158,6 +160,7 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d filename_prefix + '-py-{VERSION}.zip'.format( VERSION=bundle_version)) build_bundle(libs, bundle_version, zip_filename, + pkg_folder_prefix=pkg_folder_prefix, build_tools_version=build_tools_version) # Build .mpy bundle(s) @@ -175,7 +178,8 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d TAG=version["name"], VERSION=bundle_version)) build_bundle(libs, bundle_version, zip_filename, mpy_cross=mpy_cross, - build_tools_version=build_tools_version) + build_tools_version=build_tools_version, + pkg_folder_prefix=pkg_folder_prefix) # Build example bundle zip_filename = os.path.join(output_directory, From 346ab583d0602d4cd85c31adf58b71331f6eb3a4 Mon Sep 17 00:00:00 2001 From: sommersoft Date: Tue, 7 May 2019 18:13:12 -0500 Subject: [PATCH 2/7] Revert "Revert "Fix Mismatched arg Between Click And build_bundles()"" This reverts commit e010f5a10d4216a4a4f502c4319795982a854a17. --- circuitpython_build_tools/scripts/build_bundles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circuitpython_build_tools/scripts/build_bundles.py b/circuitpython_build_tools/scripts/build_bundles.py index ec9080d..aaa18c8 100755 --- a/circuitpython_build_tools/scripts/build_bundles.py +++ b/circuitpython_build_tools/scripts/build_bundles.py @@ -137,7 +137,7 @@ def _find_libraries(current_path, depth): @click.option('--library_location', required=True, help="Location of libraries to bundle.") @click.option('--library_depth', default=0, help="Depth of library folders. This is useful when multiple libraries are bundled together but are initially in separate subfolders.") @click.option('--package_folder_prefix', default=None, required=False, help="Prefix string used to determine package folders to bundle.") -def build_bundles(filename_prefix, output_directory, library_location, library_depth, pkg_folder_prefix): +def build_bundles(filename_prefix, output_directory, library_location, library_depth, package_folder_prefix): os.makedirs(output_directory, exist_ok=True) bundle_version = build.version_string() From f9ac09d94e068bb9c32c3e7f2773c1bef6407135 Mon Sep 17 00:00:00 2001 From: sommersoft Date: Tue, 7 May 2019 18:13:39 -0500 Subject: [PATCH 3/7] Revert "Revert "Fix arg; again..."" This reverts commit 495627bf292c9c05d2bb98079779ce808b675a06. --- circuitpython_build_tools/scripts/build_bundles.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/circuitpython_build_tools/scripts/build_bundles.py b/circuitpython_build_tools/scripts/build_bundles.py index aaa18c8..1efea22 100755 --- a/circuitpython_build_tools/scripts/build_bundles.py +++ b/circuitpython_build_tools/scripts/build_bundles.py @@ -160,7 +160,7 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d filename_prefix + '-py-{VERSION}.zip'.format( VERSION=bundle_version)) build_bundle(libs, bundle_version, zip_filename, - pkg_folder_prefix=pkg_folder_prefix, + pkg_folder_prefix=package_folder_prefix, build_tools_version=build_tools_version) # Build .mpy bundle(s) @@ -179,7 +179,7 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d VERSION=bundle_version)) build_bundle(libs, bundle_version, zip_filename, mpy_cross=mpy_cross, build_tools_version=build_tools_version, - pkg_folder_prefix=pkg_folder_prefix) + pkg_folder_prefix=package_folder_prefix) # Build example bundle zip_filename = os.path.join(output_directory, From 03373f49a9e3f7e84d9191aa0ab700ce511faa53 Mon Sep 17 00:00:00 2001 From: sommersoft Date: Tue, 7 May 2019 20:39:07 -0500 Subject: [PATCH 4/7] set @click 'package_folder_prefix' default to 'adafruit_' to maintain backwards compatibility --- circuitpython_build_tools/scripts/build_bundles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circuitpython_build_tools/scripts/build_bundles.py b/circuitpython_build_tools/scripts/build_bundles.py index 1efea22..0ae233d 100755 --- a/circuitpython_build_tools/scripts/build_bundles.py +++ b/circuitpython_build_tools/scripts/build_bundles.py @@ -136,7 +136,7 @@ def _find_libraries(current_path, depth): @click.option('--output_directory', default="bundles", help="Output location for the zip files.") @click.option('--library_location', required=True, help="Location of libraries to bundle.") @click.option('--library_depth', default=0, help="Depth of library folders. This is useful when multiple libraries are bundled together but are initially in separate subfolders.") -@click.option('--package_folder_prefix', default=None, required=False, help="Prefix string used to determine package folders to bundle.") +@click.option('--package_folder_prefix', default="adafruit_", help="Prefix string used to determine package folders to bundle.") def build_bundles(filename_prefix, output_directory, library_location, library_depth, package_folder_prefix): os.makedirs(output_directory, exist_ok=True) From 50ddabd2cc3fddd8c8e9a661fbc758392594e4d8 Mon Sep 17 00:00:00 2001 From: sommersoft Date: Sat, 11 May 2019 09:32:21 -0500 Subject: [PATCH 5/7] change 'package_folder_prefix' to a pos arg; grab examples subfolders --- circuitpython_build_tools/build.py | 16 +++++++++------- .../scripts/build_bundles.py | 17 +++++++---------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index 6bedd9e..862a5bb 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -100,7 +100,7 @@ def _munge_to_temp(original_path, temp_file, library_version): temp_file.write(line.encode("utf-8") + b"\r\n") temp_file.flush() -def library(library_path, output_directory, mpy_cross=None, example_bundle=False, pkg_folder_prefix=None): +def library(library_path, output_directory, pkg_folder_prefix, mpy_cross=None, example_bundle=False): py_files = [] package_files = [] example_files = [] @@ -115,7 +115,10 @@ def library(library_path, output_directory, mpy_cross=None, example_bundle=False # 'walked_files' while retaining subdirectory structure walked_files = [] for path in path_walk: - path_tail_idx = path[0].rfind("/") + 1 + if filename.startswith("examples"): + path_tail_idx = path[0].rfind("examples/") + 9 + else: + path_tail_idx = path[0].rfind("/") + 1 path_tail = path[0][path_tail_idx:] rel_path = "" # if this entry is the package top dir, keep it @@ -134,11 +137,10 @@ def library(library_path, output_directory, mpy_cross=None, example_bundle=False example_files.extend(files) #print("- example files: {}".format(example_files)) else: - if pkg_folder_prefix: - if (not example_bundle and - not filename.startswith(pkg_folder_prefix)): - #print("skipped path: {}".format(full_path)) - continue + if (not example_bundle and + not filename.startswith(pkg_folder_prefix)): + #print("skipped path: {}".format(full_path)) + continue if not example_bundle: package_files.extend(files) #print("- package files: {} | {}".format(filename, package_files)) diff --git a/circuitpython_build_tools/scripts/build_bundles.py b/circuitpython_build_tools/scripts/build_bundles.py index 0ae233d..65bfd7a 100755 --- a/circuitpython_build_tools/scripts/build_bundles.py +++ b/circuitpython_build_tools/scripts/build_bundles.py @@ -48,9 +48,8 @@ def add_file(bundle, src_file, zip_name): return file_sector_size -def build_bundle(libs, bundle_version, output_filename, - build_tools_version="devel", mpy_cross=None, example_bundle=False, - pkg_folder_prefix=None): +def build_bundle(libs, bundle_version, output_filename, pkg_folder_prefix, + build_tools_version="devel", mpy_cross=None, example_bundle=False): build_dir = "build-" + os.path.basename(output_filename) top_folder = os.path.basename(output_filename).replace(".zip", "") build_lib_dir = os.path.join(build_dir, top_folder, "lib") @@ -70,7 +69,7 @@ def build_bundle(libs, bundle_version, output_filename, success = True for library_path in libs: try: - build.library(library_path, build_lib_dir, pkg_folder_prefix=pkg_folder_prefix, + build.library(library_path, build_lib_dir, pkg_folder_prefix, mpy_cross=mpy_cross, example_bundle=example_bundle) except ValueError as e: print("build.library failure:", library_path) @@ -159,8 +158,7 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d zip_filename = os.path.join(output_directory, filename_prefix + '-py-{VERSION}.zip'.format( VERSION=bundle_version)) - build_bundle(libs, bundle_version, zip_filename, - pkg_folder_prefix=package_folder_prefix, + build_bundle(libs, bundle_version, zip_filename, package_folder_prefix, build_tools_version=build_tools_version) # Build .mpy bundle(s) @@ -177,13 +175,12 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d filename_prefix + '-{TAG}-mpy-{VERSION}.zip'.format( TAG=version["name"], VERSION=bundle_version)) - build_bundle(libs, bundle_version, zip_filename, mpy_cross=mpy_cross, - build_tools_version=build_tools_version, - pkg_folder_prefix=package_folder_prefix) + build_bundle(libs, bundle_version, zip_filename, package_folder_prefix, + mpy_cross=mpy_cross, build_tools_version=build_tools_version) # Build example bundle zip_filename = os.path.join(output_directory, filename_prefix + '-examples-{VERSION}.zip'.format( VERSION=bundle_version)) - build_bundle(libs, bundle_version, zip_filename, + build_bundle(libs, bundle_version, zip_filename, package_folder_prefix, build_tools_version=build_tools_version, example_bundle=True) From 0e839df058e0e2dbf81ec62664dbb06b46a78e2f Mon Sep 17 00:00:00 2001 From: sommersoft Date: Sat, 11 May 2019 09:33:41 -0500 Subject: [PATCH 6/7] README: remove build mpy-cross cmd in local testing; only used on Travis --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 5960bea..3178433 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,6 @@ locally like so: cd circuitpython-build-tools # this will be specific to your storage location python3 -m venv .env source .env/bin/activate -python3 -u -m circuitpython_build_tools.scripts.build_mpy_cross circuitpython_build_tools/data/ pip install -e . # '-e' is pip's "development" install feature circuitpython-build-bundles --filename_prefix --library_location ``` From 7f8af9417d2b6c75ff4cc652f2a6c19b497ee882 Mon Sep 17 00:00:00 2001 From: sommersoft Date: Tue, 14 May 2019 20:29:52 -0500 Subject: [PATCH 7/7] de-confuse 'package_folder_prefix' parameter naming --- circuitpython_build_tools/build.py | 4 ++-- circuitpython_build_tools/scripts/build_bundles.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index 862a5bb..d0c3d11 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -100,7 +100,7 @@ def _munge_to_temp(original_path, temp_file, library_version): temp_file.write(line.encode("utf-8") + b"\r\n") temp_file.flush() -def library(library_path, output_directory, pkg_folder_prefix, mpy_cross=None, example_bundle=False): +def library(library_path, output_directory, package_folder_prefix, mpy_cross=None, example_bundle=False): py_files = [] package_files = [] example_files = [] @@ -138,7 +138,7 @@ def library(library_path, output_directory, pkg_folder_prefix, mpy_cross=None, e #print("- example files: {}".format(example_files)) else: if (not example_bundle and - not filename.startswith(pkg_folder_prefix)): + not filename.startswith(package_folder_prefix)): #print("skipped path: {}".format(full_path)) continue if not example_bundle: diff --git a/circuitpython_build_tools/scripts/build_bundles.py b/circuitpython_build_tools/scripts/build_bundles.py index 65bfd7a..492d5a4 100755 --- a/circuitpython_build_tools/scripts/build_bundles.py +++ b/circuitpython_build_tools/scripts/build_bundles.py @@ -48,7 +48,7 @@ def add_file(bundle, src_file, zip_name): return file_sector_size -def build_bundle(libs, bundle_version, output_filename, pkg_folder_prefix, +def build_bundle(libs, bundle_version, output_filename, package_folder_prefix, build_tools_version="devel", mpy_cross=None, example_bundle=False): build_dir = "build-" + os.path.basename(output_filename) top_folder = os.path.basename(output_filename).replace(".zip", "") @@ -69,7 +69,7 @@ def build_bundle(libs, bundle_version, output_filename, pkg_folder_prefix, success = True for library_path in libs: try: - build.library(library_path, build_lib_dir, pkg_folder_prefix, + build.library(library_path, build_lib_dir, package_folder_prefix, mpy_cross=mpy_cross, example_bundle=example_bundle) except ValueError as e: print("build.library failure:", library_path)