Merge pull request #58 from adafruit/tannewt-patch-7

Stop releasing for 5.x

.gitignore

@@ -7,3 +7,4 @@ dist
 circuitpython_build_tools/data/
 .eggs
 version.py
+.env/*

.travis.yml

@@ -1,17 +1,50 @@
-sudo: false
-dist: trusty
+dist: xenial
 language: python
 python:
 - '3.6'
-script:
-- python3 -m circuitpython_build_tools.scripts.build_mpy_cross circuitpython_build_tools/data/
 
+stages:
+  - name: Tests
+    if: type = pull_request
+  - name: deploy
+    if: tag IS present
+
+jobs:
+  include:
+    - stage: Tests
+      name: "Test CircuitPython Bundle"
+      python: "3.6"
+      script:
+        - echo "Building mpy-cross" && echo "travis_fold:start:mpy-cross"
+        - python3 -u -m circuitpython_build_tools.scripts.build_mpy_cross circuitpython_build_tools/data/
+        - echo "travis_fold:end:mpy-cross"
+        - pip install -e .
+        - echo "Cloning Adafruit_CircuitPython_Bundle" && echo "travis_fold:start:clone"
+        - git clone --recurse-submodules https://github.com/adafruit/Adafruit_CircuitPython_Bundle.git
+        - echo "travis_fold:end:clone"
+        - cd Adafruit_CircuitPython_Bundle
+        - circuitpython-build-bundles --filename_prefix test-bundle --library_location libraries --library_depth 2
+
+    - stage: Tests
+      name: "Test Single Library Bundle"
+      script:
+        - echo "Building mpy-cross" && echo "travis_fold:start:mpy-cross"
+        - python3 -u -m circuitpython_build_tools.scripts.build_mpy_cross circuitpython_build_tools/data/
+        - echo "travis_fold:end:mpy-cross"
+        - pip install -e .
+        - git clone https://github.com/adafruit/Adafruit_CircuitPython_FeatherWing.git
+        - cd Adafruit_CircuitPython_FeatherWing
+        - circuitpython-build-bundles --filename_prefix test-single --library_location .
+
+    - stage: deploy
+      script:
+        - python3 -u -m circuitpython_build_tools.scripts.build_mpy_cross circuitpython_build_tools/data/
+      deploy:
+        provider: pypi
+        user: "adafruit-travis"
+        password:
+          secure: tCjXwD8YeitG0HZLxW1D1QlVv4Xbj8mfRoqW0CM9ikPp3KY1PCz6Axj0PiOcyVwKdnxcUQ0EGRl16wEqwkObrmo9MboYrPuPqN00ULmyCQCRvJa2abIN6jDoLtBuf6bcze88t0XY2LdMOcj2Udv5Iijgf95zUgE+Z6BqT9Rgche78JEOeANJ7BlAJ6nRCA4whDdG7J9s7SmFtIjKWtMxig2J3X+Qy0bZ+Armtfp9/CRvjLJ8juDrcCBSysWLnAYLS4u8e/rbSTh8YwFeoeJ1pp9qSmME5NuwScY18QmfESNSqz8wVVXtAFKdoMOCoN+/CodTxp9aB0QsXX6yOYg74ahDIaci239wgnuUqxSaeLxeSwWkkVCXWdQVuP4vgq3GZwm2yNOQ1ZjfFbXF156yv0uSVw5nuaxv0YblQTinJtL4x9hwOdPDJio3b6UT3H1ue9l1qK0LT2OSkzDgn12WmTnTfRUH3BkU6onsYsdP33PK1YhepeQnfbT1P3ikrRHIwGYb7XqcjOtJh413kid6YezCXRqccl8kAxegnqX+cQG7K9ilpZtWaVYLu4RRBJ37H4vpuOb3SV686Y62sWPUXEbI3MR1OxU+RrRr/9DCH1EFXnlYT9LF986wXFJtWuSc+pbXuxY7qduai0hn5Pft6XH31exyiOwAHBIFeYebnVM=
+        skip_cleanup: true
+        on:
+          tags: true
 
-deploy:
-  provider: pypi
-  user: "adafruit-travis"
-  password:
-    secure: tCjXwD8YeitG0HZLxW1D1QlVv4Xbj8mfRoqW0CM9ikPp3KY1PCz6Axj0PiOcyVwKdnxcUQ0EGRl16wEqwkObrmo9MboYrPuPqN00ULmyCQCRvJa2abIN6jDoLtBuf6bcze88t0XY2LdMOcj2Udv5Iijgf95zUgE+Z6BqT9Rgche78JEOeANJ7BlAJ6nRCA4whDdG7J9s7SmFtIjKWtMxig2J3X+Qy0bZ+Armtfp9/CRvjLJ8juDrcCBSysWLnAYLS4u8e/rbSTh8YwFeoeJ1pp9qSmME5NuwScY18QmfESNSqz8wVVXtAFKdoMOCoN+/CodTxp9aB0QsXX6yOYg74ahDIaci239wgnuUqxSaeLxeSwWkkVCXWdQVuP4vgq3GZwm2yNOQ1ZjfFbXF156yv0uSVw5nuaxv0YblQTinJtL4x9hwOdPDJio3b6UT3H1ue9l1qK0LT2OSkzDgn12WmTnTfRUH3BkU6onsYsdP33PK1YhepeQnfbT1P3ikrRHIwGYb7XqcjOtJh413kid6YezCXRqccl8kAxegnqX+cQG7K9ilpZtWaVYLu4RRBJ37H4vpuOb3SV686Y62sWPUXEbI3MR1OxU+RrRr/9DCH1EFXnlYT9LF986wXFJtWuSc+pbXuxY7qduai0hn5Pft6XH31exyiOwAHBIFeYebnVM=
-  skip_cleanup: true
-  on:
-    tags: true

README.md

@@ -1,6 +1,6 @@
 # Adafruit CircuitPython Build Tools
 
-[![Discord](https://img.shields.io/discord/327254708534116352.svg)](https://discord.gg/nBQh6qu)
+[![Discord](https://img.shields.io/discord/327254708534116352.svg)](https://adafru.it/discord)
 
 This repo contains build scripts used to build the
 [Adafruit CircuitPython bundle](https://github.com/adafruit/Adafruit_CircuitPython_Bundle), [CircuitPython Community bundle](https://github.com/adafruit/CircuitPython_Community_Bundle)
@@ -56,3 +56,20 @@ source .env/bin/activate
 pip install circuitpython-build-tools
 circuitpython-build-bundles --filename_prefix <output file prefix> --library_location .
 ```
+
+When making changes to `circuitpython-build-tools` itself, you can test your changes
+locally like so:
+
+```shell
+cd circuitpython-build-tools # this will be specific to your storage location
+python3 -m venv .env
+source .env/bin/activate
+pip install -e .  # '-e' is pip's "development" install feature
+circuitpython-build-bundles --filename_prefix <output file prefix> --library_location <library location>
+```
+
+## Contributing
+
+Contributions are welcome! Please read our [Code of Conduct]
+(https://github.com/adafruit/Adafruit_CircuitPython_adabot/blob/master/CODE_OF_CONDUCT.md)
+before contributing to help this project stay welcoming.

circuitpython_build_tools/build.py

@@ -3,6 +3,7 @@
 # The MIT License (MIT)
 #
 # Copyright (c) 2016 Scott Shawcroft for Adafruit Industries
+#               2018, 2019 Michael Schroeder
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -24,6 +25,7 @@
 
 import os
 import os.path
+import pathlib
 import semver
 import shutil
 import sys
@@ -31,6 +33,7 @@ import subprocess
 import tempfile
 
 IGNORE_PY = ["setup.py", "conf.py", "__init__.py"]
+GLOB_PATTERNS = ["*.py", "font5x8.bin"]
 
 def version_string(path=None, *, valid_semver=False):
     version = None
@@ -100,33 +103,52 @@ def _munge_to_temp(original_path, temp_file, library_version):
                 temp_file.write(line.encode("utf-8") + b"\r\n")
     temp_file.flush()
 
-def library(library_path, output_directory, mpy_cross=None, example_bundle=False):
+def library(library_path, output_directory, package_folder_prefix,
+            mpy_cross=None, example_bundle=False):
     py_files = []
     package_files = []
     example_files = []
     total_size = 512
-    for filename in os.listdir(library_path):
-        full_path = os.path.join(library_path, filename)
-        if os.path.isdir(full_path) and filename not in ["docs"]:
-            files = os.listdir(full_path)
-            files = filter(lambda x: x.endswith(".py") or x.startswith("font5x8.bin"), files)
-            files = map(lambda x: os.path.join(filename, x), files)
-            if filename.startswith("examples"):
-                example_files.extend(files)
-            else:
-                if not example_bundle:
-                    package_files.extend(files)
-        if (filename.endswith(".py") and
-           filename not in IGNORE_PY and
-           not example_bundle):
-            py_files.append(filename)
+
+    lib_path = pathlib.Path(library_path)
+    parent_idx = len(lib_path.parts)
+    glob_search = []
+    for pattern in GLOB_PATTERNS:
+        glob_search.extend(list(lib_path.rglob(pattern)))
+
+    for file in glob_search:
+        if file.parts[parent_idx] == "examples":
+            example_files.append(file)
+        else:
+            if not example_bundle:
+                is_package = False
+                for prefix in package_folder_prefix:
+                    if file.parts[parent_idx].startswith(prefix):
+                        is_package = True
+
+                if is_package:
+                    package_files.append(file)
+                else:
+                    if file.name in IGNORE_PY:
+                        #print("Ignoring:", file.resolve())
+                        continue
+                    if file.parent == lib_path:
+                        py_files.append(file)
 
     if len(py_files) > 1:
-        raise ValueError("Multiple top level py files not allowed. Please put them in a package "
-                         "or combine them into a single file.")
+        raise ValueError("Multiple top level py files not allowed. Please put "
+                         "them in a package or combine them into a single file.")
+
+    for fn in example_files:
+        base_dir = os.path.join(output_directory.replace("/lib", "/"),
+                                fn.relative_to(library_path).parent)
+        if not os.path.isdir(base_dir):
+            os.makedirs(base_dir)
+            total_size += 512
 
     for fn in package_files:
-        base_dir = os.path.join(output_directory, os.path.dirname(fn))
+        base_dir = os.path.join(output_directory,
+                                fn.relative_to(library_path).parent)
         if not os.path.isdir(base_dir):
             os.makedirs(base_dir)
             total_size += 512
@@ -144,16 +166,20 @@ def library(library_path, output_directory, mpy_cross=None, example_bundle=False
 
     for filename in py_files:
         full_path = os.path.join(library_path, filename)
-        output_file = os.path.join(output_directory,
-                                   filename.replace(".py", new_extension))
+        output_file = os.path.join(
+            output_directory,
+            filename.relative_to(library_path).with_suffix(new_extension)
+        )
         with tempfile.NamedTemporaryFile() as temp_file:
             _munge_to_temp(full_path, temp_file, library_version)
 
             if mpy_cross:
-                mpy_success = subprocess.call([mpy_cross,
-                                               "-o", output_file,
-                                               "-s", filename,
-                                               temp_file.name])
+                mpy_success = subprocess.call([
+                    mpy_cross,
+                    "-o", output_file,
+                    "-s", str(filename.relative_to(library_path)),
+                    temp_file.name
+                ])
                 if mpy_success != 0:
                     raise RuntimeError("mpy-cross failed on", full_path)
             else:
@@ -163,24 +189,29 @@ def library(library_path, output_directory, mpy_cross=None, example_bundle=False
         full_path = os.path.join(library_path, filename)
         with tempfile.NamedTemporaryFile() as temp_file:
             _munge_to_temp(full_path, temp_file, library_version)
-            if (not mpy_cross or
-                    os.stat(full_path).st_size == 0 or
-                    filename.endswith("__init__.py")):
-                output_file = os.path.join(output_directory, filename)
+            if not mpy_cross or os.stat(full_path).st_size == 0:
+                output_file = os.path.join(output_directory,
+                                           filename.relative_to(library_path))
                 shutil.copyfile(temp_file.name, output_file)
             else:
-                output_file = os.path.join(output_directory,
-                                           filename.replace(".py", new_extension))
-                mpy_success = subprocess.call([mpy_cross,
-                                               "-o", output_file,
-                                               "-s", filename,
-                                               temp_file.name])
+                output_file = os.path.join(
+                    output_directory,
+                    filename.relative_to(library_path).with_suffix(new_extension)
+                )
+
+                mpy_success = subprocess.call([
+                    mpy_cross,
+                    "-o", output_file,
+                    "-s", str(filename.relative_to(library_path)),
+                    temp_file.name
+                ])
                 if mpy_success != 0:
                     raise RuntimeError("mpy-cross failed on", full_path)
 
     for filename in example_files:
         full_path = os.path.join(library_path, filename)
-        output_file = os.path.join(output_directory.replace("/lib", "/"), filename)
+        output_file = os.path.join(output_directory.replace("/lib", "/"),
+                                   filename.relative_to(library_path))
         with tempfile.NamedTemporaryFile() as temp_file:
             _munge_to_temp(full_path, temp_file, library_version)
             shutil.copyfile(temp_file.name, output_file)

circuitpython_build_tools/scripts/build_bundles.py

@@ -48,7 +48,7 @@ def add_file(bundle, src_file, zip_name):
     return file_sector_size
 
 
-def build_bundle(libs, bundle_version, output_filename,
+def build_bundle(libs, bundle_version, output_filename, package_folder_prefix,
         build_tools_version="devel", mpy_cross=None, example_bundle=False):
     build_dir = "build-" + os.path.basename(output_filename)
     top_folder = os.path.basename(output_filename).replace(".zip", "")
@@ -69,8 +69,8 @@ def build_bundle(libs, bundle_version, output_filename,
     success = True
     for library_path in libs:
         try:
-            build.library(library_path, build_lib_dir, mpy_cross=mpy_cross,
-                          example_bundle=example_bundle)
+            build.library(library_path, build_lib_dir,  package_folder_prefix,
+                          mpy_cross=mpy_cross, example_bundle=example_bundle)
         except ValueError as e:
             print("build.library failure:", library_path)
             print(e)
@@ -91,7 +91,9 @@ def build_bundle(libs, bundle_version, output_filename,
             for line in versions.stdout.split(b"\n"):
                 if not line:
                     continue
-                if line.startswith(b"git@"):
+                if line.startswith(b"ssh://git@"):
+                    repo = b"https://" + line.split(b"@")[1][:-len(".git")]
+                elif line.startswith(b"git@"):
                     repo = b"https://github.com/" + line.split(b":")[1][:-len(".git")]
                 elif line.startswith(b"https:"):
                     repo = line.strip()[:-len(".git")]
@@ -135,9 +137,12 @@ def _find_libraries(current_path, depth):
 @click.option('--output_directory', default="bundles", help="Output location for the zip files.")
 @click.option('--library_location', required=True, help="Location of libraries to bundle.")
 @click.option('--library_depth', default=0, help="Depth of library folders. This is useful when multiple libraries are bundled together but are initially in separate subfolders.")
-def build_bundles(filename_prefix, output_directory, library_location, library_depth):
+@click.option('--package_folder_prefix', default="adafruit_", help="Prefix string used to determine package folders to bundle.")
+def build_bundles(filename_prefix, output_directory, library_location, library_depth, package_folder_prefix):
     os.makedirs(output_directory, exist_ok=True)
 
+    package_folder_prefix = package_folder_prefix.split(", ")
+
     bundle_version = build.version_string()
 
     libs = _find_libraries(os.path.abspath(library_location), library_depth)
@@ -157,7 +162,7 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d
     zip_filename = os.path.join(output_directory,
         filename_prefix + '-py-{VERSION}.zip'.format(
             VERSION=bundle_version))
-    build_bundle(libs, bundle_version, zip_filename,
+    build_bundle(libs, bundle_version, zip_filename, package_folder_prefix,
                  build_tools_version=build_tools_version)
 
     # Build .mpy bundle(s)
@@ -174,12 +179,12 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d
             filename_prefix + '-{TAG}-mpy-{VERSION}.zip'.format(
                 TAG=version["name"],
                 VERSION=bundle_version))
-        build_bundle(libs, bundle_version, zip_filename, mpy_cross=mpy_cross,
-                     build_tools_version=build_tools_version)
+        build_bundle(libs, bundle_version, zip_filename, package_folder_prefix,
+                     mpy_cross=mpy_cross, build_tools_version=build_tools_version)
 
     # Build example bundle
     zip_filename = os.path.join(output_directory,
         filename_prefix + '-examples-{VERSION}.zip'.format(
             VERSION=bundle_version))
-    build_bundle(libs, bundle_version, zip_filename,
+    build_bundle(libs, bundle_version, zip_filename, package_folder_prefix,
                  build_tools_version=build_tools_version, example_bundle=True)

circuitpython_build_tools/target_versions.py

@@ -25,7 +25,5 @@
 # The tag specifies which version of CircuitPython to use for mpy-cross.
 # The name is used when constructing the zip file names.
 VERSIONS = [
-    {"tag": "2.3.1", "name": "2.x"},
-    {"tag": "3.0.0", "name": "3.x"},
-    {"tag": "4.0.0-alpha.2", "name": "4.x"},
+    {"tag": "6.1.0", "name": "6.x"},
 ]