Merge pull request #58 from adafruit/tannewt-patch-7

Stop releasing for 5.x

.gitignore

@@ -7,3 +7,4 @@ dist
 circuitpython_build_tools/data/
 .eggs
 version.py
+.env/*

.travis.yml

@@ -1,12 +1,44 @@
-sudo: false
-dist: trusty
+dist: xenial
 language: python
 python:
 - '3.6'
+
+stages:
+  - name: Tests
+    if: type = pull_request
+  - name: deploy
+    if: tag IS present
+
+jobs:
+  include:
+    - stage: Tests
+      name: "Test CircuitPython Bundle"
+      python: "3.6"
       script:
-- python3 -m circuitpython_build_tools.scripts.build_mpy_cross circuitpython_build_tools/data/
+        - echo "Building mpy-cross" && echo "travis_fold:start:mpy-cross"
+        - python3 -u -m circuitpython_build_tools.scripts.build_mpy_cross circuitpython_build_tools/data/
+        - echo "travis_fold:end:mpy-cross"
+        - pip install -e .
+        - echo "Cloning Adafruit_CircuitPython_Bundle" && echo "travis_fold:start:clone"
+        - git clone --recurse-submodules https://github.com/adafruit/Adafruit_CircuitPython_Bundle.git
+        - echo "travis_fold:end:clone"
+        - cd Adafruit_CircuitPython_Bundle
+        - circuitpython-build-bundles --filename_prefix test-bundle --library_location libraries --library_depth 2
 
+    - stage: Tests
+      name: "Test Single Library Bundle"
+      script:
+        - echo "Building mpy-cross" && echo "travis_fold:start:mpy-cross"
+        - python3 -u -m circuitpython_build_tools.scripts.build_mpy_cross circuitpython_build_tools/data/
+        - echo "travis_fold:end:mpy-cross"
+        - pip install -e .
+        - git clone https://github.com/adafruit/Adafruit_CircuitPython_FeatherWing.git
+        - cd Adafruit_CircuitPython_FeatherWing
+        - circuitpython-build-bundles --filename_prefix test-single --library_location .
 
+    - stage: deploy
+      script:
+        - python3 -u -m circuitpython_build_tools.scripts.build_mpy_cross circuitpython_build_tools/data/
       deploy:
         provider: pypi
         user: "adafruit-travis"
@@ -15,3 +47,4 @@ deploy:
         skip_cleanup: true
         on:
           tags: true
+

README.md

@@ -1,6 +1,6 @@
 # Adafruit CircuitPython Build Tools
 
-[![Discord](https://img.shields.io/discord/327254708534116352.svg)](https://discord.gg/nBQh6qu)
+[![Discord](https://img.shields.io/discord/327254708534116352.svg)](https://adafru.it/discord)
 
 This repo contains build scripts used to build the
 [Adafruit CircuitPython bundle](https://github.com/adafruit/Adafruit_CircuitPython_Bundle), [CircuitPython Community bundle](https://github.com/adafruit/CircuitPython_Community_Bundle)
@@ -56,3 +56,20 @@ source .env/bin/activate
 pip install circuitpython-build-tools
 circuitpython-build-bundles --filename_prefix <output file prefix> --library_location .
 ```
+
+When making changes to `circuitpython-build-tools` itself, you can test your changes
+locally like so:
+
+```shell
+cd circuitpython-build-tools # this will be specific to your storage location
+python3 -m venv .env
+source .env/bin/activate
+pip install -e .  # '-e' is pip's "development" install feature
+circuitpython-build-bundles --filename_prefix <output file prefix> --library_location <library location>
+```
+
+## Contributing
+
+Contributions are welcome! Please read our [Code of Conduct]
+(https://github.com/adafruit/Adafruit_CircuitPython_adabot/blob/master/CODE_OF_CONDUCT.md)
+before contributing to help this project stay welcoming.

circuitpython_build_tools/build.py

@@ -3,6 +3,7 @@
 # The MIT License (MIT)
 #
 # Copyright (c) 2016 Scott Shawcroft for Adafruit Industries
+#               2018, 2019 Michael Schroeder
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -24,6 +25,7 @@
 
 import os
 import os.path
+import pathlib
 import semver
 import shutil
 import sys
@@ -31,6 +33,7 @@ import subprocess
 import tempfile
 
 IGNORE_PY = ["setup.py", "conf.py", "__init__.py"]
+GLOB_PATTERNS = ["*.py", "font5x8.bin"]
 
 def version_string(path=None, *, valid_semver=False):
     version = None
@@ -74,8 +77,12 @@ def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False):
             sys.exit(clone.returncode)
 
     current_dir = os.getcwd()
-    os.chdir("build_deps/circuitpython/mpy-cross")
-    make = subprocess.run("git fetch && git checkout {TAG} && git submodule update && make clean && make".format(TAG=circuitpython_tag), shell=True)
+    os.chdir("build_deps/circuitpython")
+    make = subprocess.run("git fetch && git checkout {TAG} && git submodule update".format(TAG=circuitpython_tag), shell=True)
+    os.chdir("tools")
+    make = subprocess.run("git submodule update --init .", shell=True)
+    os.chdir("../mpy-cross")
+    make = subprocess.run("make clean && make", shell=True)
     os.chdir(current_dir)
 
     shutil.copy("build_deps/circuitpython/mpy-cross/mpy-cross", mpy_cross_filename)
@@ -86,37 +93,65 @@ def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False):
 def _munge_to_temp(original_path, temp_file, library_version):
     with open(original_path, "rb") as original_file:
         for line in original_file:
+            if original_path.endswith(".bin"):
+                # this is solely for adafruit_framebuf/examples/font5x8.bin
+                temp_file.write(line)
+            else:
                 line = line.decode("utf-8").strip("\n")
                 if line.startswith("__version__"):
                     line = line.replace("0.0.0-auto.0", library_version)
                 temp_file.write(line.encode("utf-8") + b"\r\n")
     temp_file.flush()
 
-def library(library_path, output_directory, mpy_cross=None):
+def library(library_path, output_directory, package_folder_prefix,
+            mpy_cross=None, example_bundle=False):
     py_files = []
     package_files = []
+    example_files = []
     total_size = 512
-    for filename in os.listdir(library_path):
-        full_path = os.path.join(library_path, filename)
-        if os.path.isdir(full_path) and filename not in ["examples", "docs"]:
-            files = os.listdir(full_path)
-            files = filter(lambda x: x.endswith(".py"), files)
-            files = map(lambda x: os.path.join(filename, x), files)
-            package_files.extend(files)
-        if filename.endswith(".py") and filename not in IGNORE_PY:
-            py_files.append(filename)
+
+    lib_path = pathlib.Path(library_path)
+    parent_idx = len(lib_path.parts)
+    glob_search = []
+    for pattern in GLOB_PATTERNS:
+        glob_search.extend(list(lib_path.rglob(pattern)))
+
+    for file in glob_search:
+        if file.parts[parent_idx] == "examples":
+            example_files.append(file)
+        else:
+            if not example_bundle:
+                is_package = False
+                for prefix in package_folder_prefix:
+                    if file.parts[parent_idx].startswith(prefix):
+                        is_package = True
+
+                if is_package:
+                    package_files.append(file)
+                else:
+                    if file.name in IGNORE_PY:
+                        #print("Ignoring:", file.resolve())
+                        continue
+                    if file.parent == lib_path:
+                        py_files.append(file)
 
     if len(py_files) > 1:
-        raise ValueError("Multiple top level py files not allowed. Please put them in a package "
-                         "or combine them into a single file.")
+        raise ValueError("Multiple top level py files not allowed. Please put "
+                         "them in a package or combine them into a single file.")
 
-    if len(package_files) > 1:
-        for fn in package_files:
-            base_dir = os.path.join(output_directory, os.path.dirname(fn))
+    for fn in example_files:
+        base_dir = os.path.join(output_directory.replace("/lib", "/"),
+                                fn.relative_to(library_path).parent)
         if not os.path.isdir(base_dir):
             os.makedirs(base_dir)
             total_size += 512
 
+    for fn in package_files:
+        base_dir = os.path.join(output_directory,
+                                fn.relative_to(library_path).parent)
+        if not os.path.isdir(base_dir):
+            os.makedirs(base_dir)
+            total_size += 512
 
     new_extension = ".py"
     if mpy_cross:
@@ -131,16 +166,20 @@ def library(library_path, output_directory, mpy_cross=None):
 
     for filename in py_files:
         full_path = os.path.join(library_path, filename)
-        output_file = os.path.join(output_directory,
-                                   filename.replace(".py", new_extension))
+        output_file = os.path.join(
+            output_directory,
+            filename.relative_to(library_path).with_suffix(new_extension)
+        )
         with tempfile.NamedTemporaryFile() as temp_file:
             _munge_to_temp(full_path, temp_file, library_version)
 
             if mpy_cross:
-                mpy_success = subprocess.call([mpy_cross,
+                mpy_success = subprocess.call([
+                    mpy_cross,
                     "-o", output_file,
-                                               "-s", filename,
-                                               temp_file.name])
+                    "-s", str(filename.relative_to(library_path)),
+                    temp_file.name
+                ])
                 if mpy_success != 0:
                     raise RuntimeError("mpy-cross failed on", full_path)
             else:
@@ -150,17 +189,29 @@ def library(library_path, output_directory, mpy_cross=None):
         full_path = os.path.join(library_path, filename)
         with tempfile.NamedTemporaryFile() as temp_file:
             _munge_to_temp(full_path, temp_file, library_version)
-            if (not mpy_cross or
-                    os.stat(full_path).st_size == 0 or
-                    filename.endswith("__init__.py")):
-                output_file = os.path.join(output_directory, filename)
+            if not mpy_cross or os.stat(full_path).st_size == 0:
+                output_file = os.path.join(output_directory,
+                                           filename.relative_to(library_path))
                 shutil.copyfile(temp_file.name, output_file)
             else:
-                output_file = os.path.join(output_directory,
-                                           filename.replace(".py", new_extension))
-                mpy_success = subprocess.call([mpy_cross,
+                output_file = os.path.join(
+                    output_directory,
+                    filename.relative_to(library_path).with_suffix(new_extension)
+                )
+
+                mpy_success = subprocess.call([
+                    mpy_cross,
                     "-o", output_file,
-                                               "-s", filename,
-                                               temp_file.name])
+                    "-s", str(filename.relative_to(library_path)),
+                    temp_file.name
+                ])
                 if mpy_success != 0:
                     raise RuntimeError("mpy-cross failed on", full_path)
+
+    for filename in example_files:
+        full_path = os.path.join(library_path, filename)
+        output_file = os.path.join(output_directory.replace("/lib", "/"),
+                                   filename.relative_to(library_path))
+        with tempfile.NamedTemporaryFile() as temp_file:
+            _munge_to_temp(full_path, temp_file, library_version)
+            shutil.copyfile(temp_file.name, output_file)

circuitpython_build_tools/scripts/build_bundles.py

@@ -48,33 +48,40 @@ def add_file(bundle, src_file, zip_name):
     return file_sector_size
 
 
-def build_bundle(libs, bundle_version, output_filename,
-        build_tools_version="devel", mpy_cross=None):
+def build_bundle(libs, bundle_version, output_filename, package_folder_prefix,
+        build_tools_version="devel", mpy_cross=None, example_bundle=False):
     build_dir = "build-" + os.path.basename(output_filename)
-    build_lib_dir = os.path.join(build_dir, "lib")
+    top_folder = os.path.basename(output_filename).replace(".zip", "")
+    build_lib_dir = os.path.join(build_dir, top_folder, "lib")
+    build_example_dir = os.path.join(build_dir, top_folder, "examples")
     if os.path.isdir(build_dir):
         print("Deleting existing build.")
         shutil.rmtree(build_dir)
+    total_size = 0
+    if not example_bundle:
         os.makedirs(build_lib_dir)
+        total_size += 512
+    os.makedirs(build_example_dir)
+    total_size += 512
 
     multiple_libs = len(libs) > 1
 
     success = True
-    total_size = 512
     for library_path in libs:
         try:
-            build.library(library_path, build_lib_dir, mpy_cross=mpy_cross)
+            build.library(library_path, build_lib_dir,  package_folder_prefix,
+                          mpy_cross=mpy_cross, example_bundle=example_bundle)
         except ValueError as e:
-            print(library_path)
+            print("build.library failure:", library_path)
             print(e)
             success = False
 
     print()
     print("Generating VERSIONS")
     if multiple_libs:
-        with open(os.path.join(build_lib_dir, "VERSIONS.txt"), "w") as f:
+        with open(os.path.join(build_dir, top_folder, "VERSIONS.txt"), "w") as f:
             f.write(bundle_version + "\r\n")
-            versions = subprocess.run('git submodule foreach \"git remote get-url origin && git describe --tags\"', shell=True, stdout=subprocess.PIPE)
+            versions = subprocess.run('git submodule --quiet foreach \"git remote get-url origin && git describe --tags\"', shell=True, stdout=subprocess.PIPE, cwd=os.path.commonpath(libs))
             if versions.returncode != 0:
                 print("Failed to generate versions file. Its likely a library hasn't been "
                       "released yet.")
@@ -82,9 +89,11 @@ def build_bundle(libs, bundle_version, output_filename,
 
             repo = None
             for line in versions.stdout.split(b"\n"):
-                if line.startswith(b"Entering") or not line:
+                if not line:
                     continue
-                if line.startswith(b"git@"):
+                if line.startswith(b"ssh://git@"):
+                    repo = b"https://" + line.split(b"@")[1][:-len(".git")]
+                elif line.startswith(b"git@"):
                     repo = b"https://github.com/" + line.split(b":")[1][:-len(".git")]
                 elif line.startswith(b"https:"):
                     repo = line.strip()[:-len(".git")]
@@ -102,9 +111,9 @@ def build_bundle(libs, bundle_version, output_filename,
         build_metadata = {"build-tools-version": build_tools_version}
         bundle.comment = json.dumps(build_metadata).encode("utf-8")
         if multiple_libs:
-            total_size += add_file(bundle, "README.txt", "lib/README.txt")
-        for root, dirs, files in os.walk(build_lib_dir):
-            ziproot = root[len(build_dir + "/"):].replace("-", "_")
+            total_size += add_file(bundle, "README.txt", os.path.join(top_folder, "README.txt"))
+        for root, dirs, files in os.walk(build_dir):
+            ziproot = root[len(build_dir + "/"):]
             for filename in files:
                 total_size += add_file(bundle, os.path.join(root, filename),
                                        os.path.join(ziproot, filename.replace("-", "_")))
@@ -128,9 +137,12 @@ def _find_libraries(current_path, depth):
 @click.option('--output_directory', default="bundles", help="Output location for the zip files.")
 @click.option('--library_location', required=True, help="Location of libraries to bundle.")
 @click.option('--library_depth', default=0, help="Depth of library folders. This is useful when multiple libraries are bundled together but are initially in separate subfolders.")
-def build_bundles(filename_prefix, output_directory, library_location, library_depth):
+@click.option('--package_folder_prefix', default="adafruit_", help="Prefix string used to determine package folders to bundle.")
+def build_bundles(filename_prefix, output_directory, library_location, library_depth, package_folder_prefix):
     os.makedirs(output_directory, exist_ok=True)
 
+    package_folder_prefix = package_folder_prefix.split(", ")
+
     bundle_version = build.version_string()
 
     libs = _find_libraries(os.path.abspath(library_location), library_depth)
@@ -146,11 +158,14 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d
     with open(build_tools_fn, "w") as f:
         f.write(build_tools_version)
 
+    # Build raw source .py bundle
     zip_filename = os.path.join(output_directory,
         filename_prefix + '-py-{VERSION}.zip'.format(
             VERSION=bundle_version))
-    build_bundle(libs, bundle_version, zip_filename,
+    build_bundle(libs, bundle_version, zip_filename, package_folder_prefix,
                  build_tools_version=build_tools_version)
+
+    # Build .mpy bundle(s)
     os.makedirs("build_deps", exist_ok=True)
     for version in target_versions.VERSIONS:
         # Use prebuilt mpy-cross on Travis, otherwise build our own.
@@ -164,5 +179,12 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d
             filename_prefix + '-{TAG}-mpy-{VERSION}.zip'.format(
                 TAG=version["name"],
                 VERSION=bundle_version))
-        build_bundle(libs, bundle_version, zip_filename, mpy_cross=mpy_cross,
-                     build_tools_version=build_tools_version)
+        build_bundle(libs, bundle_version, zip_filename, package_folder_prefix,
+                     mpy_cross=mpy_cross, build_tools_version=build_tools_version)
+
+    # Build example bundle
+    zip_filename = os.path.join(output_directory,
+        filename_prefix + '-examples-{VERSION}.zip'.format(
+            VERSION=bundle_version))
+    build_bundle(libs, bundle_version, zip_filename, package_folder_prefix,
+                 build_tools_version=build_tools_version, example_bundle=True)

circuitpython_build_tools/target_versions.py

@@ -22,7 +22,8 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+# The tag specifies which version of CircuitPython to use for mpy-cross.
+# The name is used when constructing the zip file names.
 VERSIONS = [
-    {"tag": "2.3.1", "name": "2.3.1"},
-    {"tag": "3.0.0-beta.0", "name": "3.0.0-beta.0"},
+    {"tag": "6.1.0", "name": "6.x"},
 ]

requirements.txt

@@ -1,2 +1,3 @@
 Click
 semver
+wheel