Merge pull request #58 from adafruit/tannewt-patch-7

Stop releasing for 5.x

.gitignore

@@ -7,3 +7,4 @@ dist
 circuitpython_build_tools/data/
 .eggs
 version.py
+.env/*

.travis.yml

@@ -1,17 +1,50 @@
-sudo: false
+dist: xenial
-dist: trusty
 language: python
 python:
 - '3.6'
-script:
-- python3 -m circuitpython_build_tools.scripts.build_mpy_cross circuitpython_build_tools/data/
 
+stages:
+  - name: Tests
+    if: type = pull_request
+  - name: deploy
+    if: tag IS present
+
+jobs:
+  include:
+    - stage: Tests
+      name: "Test CircuitPython Bundle"
+      python: "3.6"
+      script:
+        - echo "Building mpy-cross" && echo "travis_fold:start:mpy-cross"
+        - python3 -u -m circuitpython_build_tools.scripts.build_mpy_cross circuitpython_build_tools/data/
+        - echo "travis_fold:end:mpy-cross"
+        - pip install -e .
+        - echo "Cloning Adafruit_CircuitPython_Bundle" && echo "travis_fold:start:clone"
+        - git clone --recurse-submodules https://github.com/adafruit/Adafruit_CircuitPython_Bundle.git
+        - echo "travis_fold:end:clone"
+        - cd Adafruit_CircuitPython_Bundle
+        - circuitpython-build-bundles --filename_prefix test-bundle --library_location libraries --library_depth 2
+
+    - stage: Tests
+      name: "Test Single Library Bundle"
+      script:
+        - echo "Building mpy-cross" && echo "travis_fold:start:mpy-cross"
+        - python3 -u -m circuitpython_build_tools.scripts.build_mpy_cross circuitpython_build_tools/data/
+        - echo "travis_fold:end:mpy-cross"
+        - pip install -e .
+        - git clone https://github.com/adafruit/Adafruit_CircuitPython_FeatherWing.git
+        - cd Adafruit_CircuitPython_FeatherWing
+        - circuitpython-build-bundles --filename_prefix test-single --library_location .
+
+    - stage: deploy
+      script:
+        - python3 -u -m circuitpython_build_tools.scripts.build_mpy_cross circuitpython_build_tools/data/
+      deploy:
+        provider: pypi
+        user: "adafruit-travis"
+        password:
+          secure: tCjXwD8YeitG0HZLxW1D1QlVv4Xbj8mfRoqW0CM9ikPp3KY1PCz6Axj0PiOcyVwKdnxcUQ0EGRl16wEqwkObrmo9MboYrPuPqN00ULmyCQCRvJa2abIN6jDoLtBuf6bcze88t0XY2LdMOcj2Udv5Iijgf95zUgE+Z6BqT9Rgche78JEOeANJ7BlAJ6nRCA4whDdG7J9s7SmFtIjKWtMxig2J3X+Qy0bZ+Armtfp9/CRvjLJ8juDrcCBSysWLnAYLS4u8e/rbSTh8YwFeoeJ1pp9qSmME5NuwScY18QmfESNSqz8wVVXtAFKdoMOCoN+/CodTxp9aB0QsXX6yOYg74ahDIaci239wgnuUqxSaeLxeSwWkkVCXWdQVuP4vgq3GZwm2yNOQ1ZjfFbXF156yv0uSVw5nuaxv0YblQTinJtL4x9hwOdPDJio3b6UT3H1ue9l1qK0LT2OSkzDgn12WmTnTfRUH3BkU6onsYsdP33PK1YhepeQnfbT1P3ikrRHIwGYb7XqcjOtJh413kid6YezCXRqccl8kAxegnqX+cQG7K9ilpZtWaVYLu4RRBJ37H4vpuOb3SV686Y62sWPUXEbI3MR1OxU+RrRr/9DCH1EFXnlYT9LF986wXFJtWuSc+pbXuxY7qduai0hn5Pft6XH31exyiOwAHBIFeYebnVM=
+        skip_cleanup: true
+        on:
+          tags: true
 
-deploy:
-  provider: pypi
-  user: "adafruit-travis"
-  password:
-    secure: tCjXwD8YeitG0HZLxW1D1QlVv4Xbj8mfRoqW0CM9ikPp3KY1PCz6Axj0PiOcyVwKdnxcUQ0EGRl16wEqwkObrmo9MboYrPuPqN00ULmyCQCRvJa2abIN6jDoLtBuf6bcze88t0XY2LdMOcj2Udv5Iijgf95zUgE+Z6BqT9Rgche78JEOeANJ7BlAJ6nRCA4whDdG7J9s7SmFtIjKWtMxig2J3X+Qy0bZ+Armtfp9/CRvjLJ8juDrcCBSysWLnAYLS4u8e/rbSTh8YwFeoeJ1pp9qSmME5NuwScY18QmfESNSqz8wVVXtAFKdoMOCoN+/CodTxp9aB0QsXX6yOYg74ahDIaci239wgnuUqxSaeLxeSwWkkVCXWdQVuP4vgq3GZwm2yNOQ1ZjfFbXF156yv0uSVw5nuaxv0YblQTinJtL4x9hwOdPDJio3b6UT3H1ue9l1qK0LT2OSkzDgn12WmTnTfRUH3BkU6onsYsdP33PK1YhepeQnfbT1P3ikrRHIwGYb7XqcjOtJh413kid6YezCXRqccl8kAxegnqX+cQG7K9ilpZtWaVYLu4RRBJ37H4vpuOb3SV686Y62sWPUXEbI3MR1OxU+RrRr/9DCH1EFXnlYT9LF986wXFJtWuSc+pbXuxY7qduai0hn5Pft6XH31exyiOwAHBIFeYebnVM=
-  skip_cleanup: true
-  on:
-    tags: true

README.md

@@ -1,6 +1,6 @@
 # Adafruit CircuitPython Build Tools
 
-[![Discord](https://img.shields.io/discord/327254708534116352.svg)](https://discord.gg/nBQh6qu)
+[![Discord](https://img.shields.io/discord/327254708534116352.svg)](https://adafru.it/discord)
 
 This repo contains build scripts used to build the
 [Adafruit CircuitPython bundle](https://github.com/adafruit/Adafruit_CircuitPython_Bundle), [CircuitPython Community bundle](https://github.com/adafruit/CircuitPython_Community_Bundle)
@@ -56,3 +56,20 @@ source .env/bin/activate
 pip install circuitpython-build-tools
 circuitpython-build-bundles --filename_prefix <output file prefix> --library_location .
 ```
+
+When making changes to `circuitpython-build-tools` itself, you can test your changes
+locally like so:
+
+```shell
+cd circuitpython-build-tools # this will be specific to your storage location
+python3 -m venv .env
+source .env/bin/activate
+pip install -e .  # '-e' is pip's "development" install feature
+circuitpython-build-bundles --filename_prefix <output file prefix> --library_location <library location>
+```
+
+## Contributing
+
+Contributions are welcome! Please read our [Code of Conduct]
+(https://github.com/adafruit/Adafruit_CircuitPython_adabot/blob/master/CODE_OF_CONDUCT.md)
+before contributing to help this project stay welcoming.

circuitpython_build_tools/build.py

@@ -3,6 +3,7 @@
 # The MIT License (MIT)
 #
 # Copyright (c) 2016 Scott Shawcroft for Adafruit Industries
+#               2018, 2019 Michael Schroeder
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -24,6 +25,7 @@
 
 import os
 import os.path
+import pathlib
 import semver
 import shutil
 import sys
@@ -31,6 +33,7 @@ import subprocess
 import tempfile
 
 IGNORE_PY = ["setup.py", "conf.py", "__init__.py"]
+GLOB_PATTERNS = ["*.py", "font5x8.bin"]
 
 def version_string(path=None, *, valid_semver=False):
     version = None
@@ -90,39 +93,62 @@ def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False):
 def _munge_to_temp(original_path, temp_file, library_version):
     with open(original_path, "rb") as original_file:
         for line in original_file:
-            line = line.decode("utf-8").strip("\n")
+            if original_path.endswith(".bin"):
-            if line.startswith("__version__"):
+                # this is solely for adafruit_framebuf/examples/font5x8.bin
-                line = line.replace("0.0.0-auto.0", library_version)
+                temp_file.write(line)
-            temp_file.write(line.encode("utf-8") + b"\r\n")
+            else:
+                line = line.decode("utf-8").strip("\n")
+                if line.startswith("__version__"):
+                    line = line.replace("0.0.0-auto.0", library_version)
+                temp_file.write(line.encode("utf-8") + b"\r\n")
     temp_file.flush()
 
-def library(library_path, output_directory, mpy_cross=None, example_bundle=False):
+def library(library_path, output_directory, package_folder_prefix,
+            mpy_cross=None, example_bundle=False):
     py_files = []
     package_files = []
     example_files = []
     total_size = 512
-    for filename in os.listdir(library_path):
+
-        full_path = os.path.join(library_path, filename)
+    lib_path = pathlib.Path(library_path)
-        if os.path.isdir(full_path) and filename not in ["docs"]:
+    parent_idx = len(lib_path.parts)
-            files = os.listdir(full_path)
+    glob_search = []
-            files = filter(lambda x: x.endswith(".py"), files)
+    for pattern in GLOB_PATTERNS:
-            files = map(lambda x: os.path.join(filename, x), files)
+        glob_search.extend(list(lib_path.rglob(pattern)))
-            if filename.startswith("examples"):
+
-                example_files.extend(files)
+    for file in glob_search:
-            else:
+        if file.parts[parent_idx] == "examples":
-                if not example_bundle:
+            example_files.append(file)
-                    package_files.extend(files)
+        else:
-        if (filename.endswith(".py") and
+            if not example_bundle:
-           filename not in IGNORE_PY and
+                is_package = False
-           not example_bundle):
+                for prefix in package_folder_prefix:
-            py_files.append(filename)
+                    if file.parts[parent_idx].startswith(prefix):
+                        is_package = True
+
+                if is_package:
+                    package_files.append(file)
+                else:
+                    if file.name in IGNORE_PY:
+                        #print("Ignoring:", file.resolve())
+                        continue
+                    if file.parent == lib_path:
+                        py_files.append(file)
 
     if len(py_files) > 1:
-        raise ValueError("Multiple top level py files not allowed. Please put them in a package "
+        raise ValueError("Multiple top level py files not allowed. Please put "
-                         "or combine them into a single file.")
+                         "them in a package or combine them into a single file.")
+
+    for fn in example_files:
+        base_dir = os.path.join(output_directory.replace("/lib", "/"),
+                                fn.relative_to(library_path).parent)
+        if not os.path.isdir(base_dir):
+            os.makedirs(base_dir)
+            total_size += 512
 
     for fn in package_files:
-        base_dir = os.path.join(output_directory, os.path.dirname(fn))
+        base_dir = os.path.join(output_directory,
+                                fn.relative_to(library_path).parent)
         if not os.path.isdir(base_dir):
             os.makedirs(base_dir)
             total_size += 512
@@ -140,16 +166,20 @@ def library(library_path, output_directory, mpy_cross=None, example_bundle=False
 
     for filename in py_files:
         full_path = os.path.join(library_path, filename)
-        output_file = os.path.join(output_directory,
+        output_file = os.path.join(
-                                   filename.replace(".py", new_extension))
+            output_directory,
+            filename.relative_to(library_path).with_suffix(new_extension)
+        )
         with tempfile.NamedTemporaryFile() as temp_file:
             _munge_to_temp(full_path, temp_file, library_version)
 
             if mpy_cross:
-                mpy_success = subprocess.call([mpy_cross,
+                mpy_success = subprocess.call([
-                                               "-o", output_file,
+                    mpy_cross,
-                                               "-s", filename,
+                    "-o", output_file,
-                                               temp_file.name])
+                    "-s", str(filename.relative_to(library_path)),
+                    temp_file.name
+                ])
                 if mpy_success != 0:
                     raise RuntimeError("mpy-cross failed on", full_path)
             else:
@@ -159,24 +189,29 @@ def library(library_path, output_directory, mpy_cross=None, example_bundle=False
         full_path = os.path.join(library_path, filename)
         with tempfile.NamedTemporaryFile() as temp_file:
             _munge_to_temp(full_path, temp_file, library_version)
-            if (not mpy_cross or
+            if not mpy_cross or os.stat(full_path).st_size == 0:
-                    os.stat(full_path).st_size == 0 or
+                output_file = os.path.join(output_directory,
-                    filename.endswith("__init__.py")):
+                                           filename.relative_to(library_path))
-                output_file = os.path.join(output_directory, filename)
                 shutil.copyfile(temp_file.name, output_file)
             else:
-                output_file = os.path.join(output_directory,
+                output_file = os.path.join(
-                                           filename.replace(".py", new_extension))
+                    output_directory,
-                mpy_success = subprocess.call([mpy_cross,
+                    filename.relative_to(library_path).with_suffix(new_extension)
-                                               "-o", output_file,
+                )
-                                               "-s", filename,
+
-                                               temp_file.name])
+                mpy_success = subprocess.call([
+                    mpy_cross,
+                    "-o", output_file,
+                    "-s", str(filename.relative_to(library_path)),
+                    temp_file.name
+                ])
                 if mpy_success != 0:
                     raise RuntimeError("mpy-cross failed on", full_path)
 
     for filename in example_files:
         full_path = os.path.join(library_path, filename)
-        output_file = os.path.join(output_directory.replace("/lib", "/"), filename)
+        output_file = os.path.join(output_directory.replace("/lib", "/"),
+                                   filename.relative_to(library_path))
         with tempfile.NamedTemporaryFile() as temp_file:
             _munge_to_temp(full_path, temp_file, library_version)
             shutil.copyfile(temp_file.name, output_file)

circuitpython_build_tools/scripts/build_bundles.py

@@ -48,11 +48,12 @@ def add_file(bundle, src_file, zip_name):
     return file_sector_size
 
 
-def build_bundle(libs, bundle_version, output_filename,
+def build_bundle(libs, bundle_version, output_filename, package_folder_prefix,
         build_tools_version="devel", mpy_cross=None, example_bundle=False):
     build_dir = "build-" + os.path.basename(output_filename)
-    build_lib_dir = os.path.join(build_dir, build_dir.replace(".zip", ""), "lib")
+    top_folder = os.path.basename(output_filename).replace(".zip", "")
-    build_example_dir = os.path.join(build_dir, build_dir.replace(".zip", ""), "examples")
+    build_lib_dir = os.path.join(build_dir, top_folder, "lib")
+    build_example_dir = os.path.join(build_dir, top_folder, "examples")
     if os.path.isdir(build_dir):
         print("Deleting existing build.")
         shutil.rmtree(build_dir)
@@ -68,8 +69,8 @@ def build_bundle(libs, bundle_version, output_filename,
     success = True
     for library_path in libs:
         try:
-            build.library(library_path, build_lib_dir, mpy_cross=mpy_cross,
+            build.library(library_path, build_lib_dir,  package_folder_prefix,
-                          example_bundle=example_bundle)
+                          mpy_cross=mpy_cross, example_bundle=example_bundle)
         except ValueError as e:
             print("build.library failure:", library_path)
             print(e)
@@ -78,9 +79,9 @@ def build_bundle(libs, bundle_version, output_filename,
     print()
     print("Generating VERSIONS")
     if multiple_libs:
-        with open(os.path.join(build_dir, build_dir.replace(".zip", ""), "VERSIONS.txt"), "w") as f:
+        with open(os.path.join(build_dir, top_folder, "VERSIONS.txt"), "w") as f:
             f.write(bundle_version + "\r\n")
-            versions = subprocess.run('git submodule foreach \"git remote get-url origin && git describe --tags\"', shell=True, stdout=subprocess.PIPE, cwd=os.path.commonpath(libs))
+            versions = subprocess.run('git submodule --quiet foreach \"git remote get-url origin && git describe --tags\"', shell=True, stdout=subprocess.PIPE, cwd=os.path.commonpath(libs))
             if versions.returncode != 0:
                 print("Failed to generate versions file. Its likely a library hasn't been "
                       "released yet.")
@@ -88,9 +89,11 @@ def build_bundle(libs, bundle_version, output_filename,
 
             repo = None
             for line in versions.stdout.split(b"\n"):
-                if line.startswith(b"Entering") or not line:
+                if not line:
                     continue
-                if line.startswith(b"git@"):
+                if line.startswith(b"ssh://git@"):
+                    repo = b"https://" + line.split(b"@")[1][:-len(".git")]
+                elif line.startswith(b"git@"):
                     repo = b"https://github.com/" + line.split(b":")[1][:-len(".git")]
                 elif line.startswith(b"https:"):
                     repo = line.strip()[:-len(".git")]
@@ -108,10 +111,9 @@ def build_bundle(libs, bundle_version, output_filename,
         build_metadata = {"build-tools-version": build_tools_version}
         bundle.comment = json.dumps(build_metadata).encode("utf-8")
         if multiple_libs:
-            readme_zip_dir = build_dir.replace(".zip", "")
+            total_size += add_file(bundle, "README.txt", os.path.join(top_folder, "README.txt"))
-            total_size += add_file(bundle, "README.txt", os.path.join(readme_zip_dir.replace("-", "_"), "README.txt"))
         for root, dirs, files in os.walk(build_dir):
-            ziproot = root[len(build_dir + "/"):].replace("-", "_")
+            ziproot = root[len(build_dir + "/"):]
             for filename in files:
                 total_size += add_file(bundle, os.path.join(root, filename),
                                        os.path.join(ziproot, filename.replace("-", "_")))
@@ -135,9 +137,12 @@ def _find_libraries(current_path, depth):
 @click.option('--output_directory', default="bundles", help="Output location for the zip files.")
 @click.option('--library_location', required=True, help="Location of libraries to bundle.")
 @click.option('--library_depth', default=0, help="Depth of library folders. This is useful when multiple libraries are bundled together but are initially in separate subfolders.")
-def build_bundles(filename_prefix, output_directory, library_location, library_depth):
+@click.option('--package_folder_prefix', default="adafruit_", help="Prefix string used to determine package folders to bundle.")
+def build_bundles(filename_prefix, output_directory, library_location, library_depth, package_folder_prefix):
     os.makedirs(output_directory, exist_ok=True)
 
+    package_folder_prefix = package_folder_prefix.split(", ")
+
     bundle_version = build.version_string()
 
     libs = _find_libraries(os.path.abspath(library_location), library_depth)
@@ -157,7 +162,7 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d
     zip_filename = os.path.join(output_directory,
         filename_prefix + '-py-{VERSION}.zip'.format(
             VERSION=bundle_version))
-    build_bundle(libs, bundle_version, zip_filename,
+    build_bundle(libs, bundle_version, zip_filename, package_folder_prefix,
                  build_tools_version=build_tools_version)
 
     # Build .mpy bundle(s)
@@ -174,12 +179,12 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d
             filename_prefix + '-{TAG}-mpy-{VERSION}.zip'.format(
                 TAG=version["name"],
                 VERSION=bundle_version))
-        build_bundle(libs, bundle_version, zip_filename, mpy_cross=mpy_cross,
+        build_bundle(libs, bundle_version, zip_filename, package_folder_prefix,
-                     build_tools_version=build_tools_version)
+                     mpy_cross=mpy_cross, build_tools_version=build_tools_version)
 
     # Build example bundle
     zip_filename = os.path.join(output_directory,
         filename_prefix + '-examples-{VERSION}.zip'.format(
             VERSION=bundle_version))
-    build_bundle(libs, bundle_version, zip_filename,
+    build_bundle(libs, bundle_version, zip_filename, package_folder_prefix,
                  build_tools_version=build_tools_version, example_bundle=True)

circuitpython_build_tools/target_versions.py

@@ -25,7 +25,5 @@
 # The tag specifies which version of CircuitPython to use for mpy-cross.
 # The name is used when constructing the zip file names.
 VERSIONS = [
-    {"tag": "2.3.1", "name": "2.x"},
+    {"tag": "6.1.0", "name": "6.x"},
-    {"tag": "3.0.0", "name": "3.x"},
-    {"tag": "4.0.0-alpha.2", "name": "4.x"},
 ]

requirements.txt

@@ -1,2 +1,3 @@
 Click
 semver
+wheel