Add circuitpython-mpy-cross

This wraps the process of building or downloading mpy-cross for a specified
version, then running it with specified arguments.

e.g.,
```
$ circuitpython-mpy-cross --circuitpython-version 9.x -- --version
CircuitPython 9.0.0-alpha.2 on 2023-10-27; mpy-cross emitting mpy v6.1
```

.github/workflows/build.yml

@@ -0,0 +1,55 @@
+# SPDX-FileCopyrightText: 2017 Scott Shawcroft, written for Adafruit Industries
+# SPDX-FileCopyrightText: 2021 James Carr
+#
+# SPDX-License-Identifier: MIT
+
+name: Build CI
+
+on: [pull_request, push]
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Dump GitHub context
+      env:
+        GITHUB_CONTEXT: ${{ toJson(github) }}
+      run: echo "$GITHUB_CONTEXT"
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.10"
+    - name: Versions
+      run: |
+        python3 --version
+    - name: Checkout Current Repo
+      uses: actions/checkout@v4
+      with:
+        filter: 'blob:none'
+        depth: 0
+    - name: Install requirements
+      run: |
+        sudo apt-get update
+        sudo apt-get install libudev-dev libusb-1.0
+        sudo apt-get install -y gettext
+        pip install -r requirements.txt
+    - name: Library version
+      run: git describe --dirty --always --tags
+    - name: Install package locally
+      run: pip install -e .
+    - name: Test building single package
+      run: |
+        git clone https://github.com/adafruit/Adafruit_CircuitPython_FeatherWing.git
+        cd Adafruit_CircuitPython_FeatherWing
+        circuitpython-build-bundles --filename_prefix test-single --library_location .
+    - name: Test building bundle
+      run: |
+        # Use the community bundle because it's smaller and faster
+        git clone --recurse-submodules https://github.com/adafruit/CircuitPython_Community_Bundle.git
+        cd CircuitPython_Community_Bundle
+        circuitpython-build-bundles --filename_prefix test-bundle --library_location libraries --library_depth 2
+    - name: Build Python package
+      run: |
+        pip install --upgrade setuptools wheel twine readme_renderer testresources
+        python setup.py sdist
+        twine check dist/*

.github/workflows/release.yml

@@ -0,0 +1,34 @@
+# SPDX-FileCopyrightText: 2017 Scott Shawcroft, written for Adafruit Industries
+# SPDX-FileCopyrightText: 2021 James Carr
+#
+# SPDX-License-Identifier: MIT
+
+name: Release Actions
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  upload-pypi:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        filter: 'blob:none'
+        depth: 0
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine
+    - name: Build and publish
+      env:
+        TWINE_USERNAME: ${{ secrets.pypi_username }}
+        TWINE_PASSWORD: ${{ secrets.pypi_password }}
+      run: |
+        python setup.py sdist
+        twine upload dist/*

.gitignore

@@ -8,3 +8,5 @@ circuitpython_build_tools/data/
 .eggs
 version.py
 .env/*
+.DS_Store
+.idea/*

.travis.yml

@@ -1,50 +0,0 @@
-dist: xenial
-language: python
-python:
-- '3.6'
-
-stages:
-  - name: Tests
-    if: type = pull_request
-  - name: deploy
-    if: tag IS present
-
-jobs:
-  include:
-    - stage: Tests
-      name: "Test CircuitPython Bundle"
-      python: "3.6"
-      script:
-        - echo "Building mpy-cross" && echo "travis_fold:start:mpy-cross"
-        - python3 -u -m circuitpython_build_tools.scripts.build_mpy_cross circuitpython_build_tools/data/
-        - echo "travis_fold:end:mpy-cross"
-        - pip install -e .
-        - echo "Cloning Adafruit_CircuitPython_Bundle" && echo "travis_fold:start:clone"
-        - git clone --recurse-submodules https://github.com/adafruit/Adafruit_CircuitPython_Bundle.git
-        - echo "travis_fold:end:clone"
-        - cd Adafruit_CircuitPython_Bundle
-        - circuitpython-build-bundles --filename_prefix test-bundle --library_location libraries --library_depth 2
-
-    - stage: Tests
-      name: "Test Single Library Bundle"
-      script:
-        - echo "Building mpy-cross" && echo "travis_fold:start:mpy-cross"
-        - python3 -u -m circuitpython_build_tools.scripts.build_mpy_cross circuitpython_build_tools/data/
-        - echo "travis_fold:end:mpy-cross"
-        - pip install -e .
-        - git clone https://github.com/adafruit/Adafruit_CircuitPython_FeatherWing.git
-        - cd Adafruit_CircuitPython_FeatherWing
-        - circuitpython-build-bundles --filename_prefix test-single --library_location .
-
-    - stage: deploy
-      script:
-        - python3 -u -m circuitpython_build_tools.scripts.build_mpy_cross circuitpython_build_tools/data/
-      deploy:
-        provider: pypi
-        user: "adafruit-travis"
-        password:
-          secure: tCjXwD8YeitG0HZLxW1D1QlVv4Xbj8mfRoqW0CM9ikPp3KY1PCz6Axj0PiOcyVwKdnxcUQ0EGRl16wEqwkObrmo9MboYrPuPqN00ULmyCQCRvJa2abIN6jDoLtBuf6bcze88t0XY2LdMOcj2Udv5Iijgf95zUgE+Z6BqT9Rgche78JEOeANJ7BlAJ6nRCA4whDdG7J9s7SmFtIjKWtMxig2J3X+Qy0bZ+Armtfp9/CRvjLJ8juDrcCBSysWLnAYLS4u8e/rbSTh8YwFeoeJ1pp9qSmME5NuwScY18QmfESNSqz8wVVXtAFKdoMOCoN+/CodTxp9aB0QsXX6yOYg74ahDIaci239wgnuUqxSaeLxeSwWkkVCXWdQVuP4vgq3GZwm2yNOQ1ZjfFbXF156yv0uSVw5nuaxv0YblQTinJtL4x9hwOdPDJio3b6UT3H1ue9l1qK0LT2OSkzDgn12WmTnTfRUH3BkU6onsYsdP33PK1YhepeQnfbT1P3ikrRHIwGYb7XqcjOtJh413kid6YezCXRqccl8kAxegnqX+cQG7K9ilpZtWaVYLu4RRBJ37H4vpuOb3SV686Y62sWPUXEbI3MR1OxU+RrRr/9DCH1EFXnlYT9LF986wXFJtWuSc+pbXuxY7qduai0hn5Pft6XH31exyiOwAHBIFeYebnVM=
-        skip_cleanup: true
-        on:
-          tags: true
-

README.md

@@ -4,40 +4,20 @@
 
 This repo contains build scripts used to build the
 [Adafruit CircuitPython bundle](https://github.com/adafruit/Adafruit_CircuitPython_Bundle), [CircuitPython Community bundle](https://github.com/adafruit/CircuitPython_Community_Bundle)
-and individual library release zips. Its focused on Travis CI support but will also work locally
+and individual library release zips. Its focused on Github Actions support but will also work locally
 when a gcc compiler is present.
 
-The pip package includes mpy-crosses that run on Travis. When building locally, the scripts will
+The scripts will either fetch a pre-built mpy-cross from s3 or
 automatically clone the [CircuitPython repo](https://github.com/adafruit/circuitpython) and attempt
-to build mpy-crosses. You'll need some version of gcc for this to work.
+to build mpy-cross. You'll need some version of gcc for this to work.
 
 ## Setting up libraries
 
-These build tools are intended for use with [Travis CI](https://travis-ci.org)
-to automatically build .mpy files and zip them up for CircuitPython when a new
-tagged release is created.  To add support to a repo you need to:
-
-  1. Use the [CircuitPython cookiecutter](https://github.com/adafruit/cookiecutter-adafruit-circuitpython) to generate .travis.yml.
-  2. For adafruit repositories, simply give the CircuitPythonLibrarians team
-     write access to the repo and Adabot will do the rest.
-
-     Otherwise, go to travis-ci.org and find the repository (it needs to be
-     setup to access your github account, and your github account needs access
-     to write to the repo).  Flip the 'ON' switch on for Travis and the repo,
-     see the Travis docs for more details: https://docs.travis-ci.com/user/getting-started/
-  3. Get a GitHub 'personal access token' which has at least 'public_repo' or
-     'repo' scope: https://help.github.com/articles/creating-an-access-token-for-command-line-use/
-     Keep this token safe and secure!  Anyone with the token will be able to
-     access and write to your GitHub repositories.  Travis will use the token
-     to attach the .mpy files to the release.
-  4. In the Travis CI settings for the repository that was enabled find the
-     environment variable editing page: https://docs.travis-ci.com/user/environment-variables/#Defining-Variables-in-Repository-Settings
-     Add an environment variable named GITHUB_TOKEN and set it to the value
-     of the GitHub personal access token above.  Keep 'Display value in build
-     log' flipped off.
-  5. That's it!  Tag a release and Travis should go to work to add zipped .mpy files
-     to the release.  It takes about a 2-3 minutes for a worker to spin up,
-     build mpy-cross, and add the binaries to the release.
+These build tools automatically build .mpy files and zip them up for
+CircuitPython when a new tagged release is created. To add support to a repo
+you need to use the [CircuitPython
+cookiecutter](https://github.com/adafruit/cookiecutter-adafruit-circuitpython)
+to generate `.github/workflows/*.yml`.
 
 The bundle build will produce one zip file for every major CircuitPython
 release supported containing compatible mpy files and a zip with human readable py files.
@@ -71,5 +51,5 @@ circuitpython-build-bundles --filename_prefix <output file prefix> --library_loc
 ## Contributing
 
 Contributions are welcome! Please read our [Code of Conduct]
-(https://github.com/adafruit/Adafruit_CircuitPython_adabot/blob/master/CODE_OF_CONDUCT.md)
+(https://github.com/adafruit/Adafruit\_CircuitPython\_adabot/blob/master/CODE\_OF\_CONDUCT.md)
 before contributing to help this project stay welcoming.

circuitpython_build_tools/build.py

@@ -4,6 +4,7 @@
 #
 # Copyright (c) 2016 Scott Shawcroft for Adafruit Industries
 #               2018, 2019 Michael Schroeder
+#               2021 James Carr
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -23,17 +24,83 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+import functools
+import multiprocessing
 import os
 import os.path
+import platform
 import pathlib
+import re
+import requests
 import semver
 import shutil
+import stat
 import sys
 import subprocess
 import tempfile
+import platformdirs
+
+@functools.cache
+def _git_version():                                                 
+    version_str = subprocess.check_output(["git", "--version"], encoding="ascii", errors="replace")                    
+    version_str = re.search("([0-9]\.*)*[0-9]", version_str).group(0)
+    return tuple(int(part) for part in version_str.split("."))
+                 
+def git_filter_arg():
+    clone_supports_filter = (                    
+        False if "NO_USE_CLONE_FILTER" in os.environ else _git_version() >= (2, 36, 0)
+    )                                                                            
+                                                                                 
+    if clone_supports_filter:                                                
+        return ["--filter=blob:none"]
+    else:                                                              
+        return []
+
+# pyproject.toml `py_modules` values that are incorrect. These should all have PRs filed!
+# and should be removed when the fixed version is incorporated in its respective bundle.
+
+pyproject_py_modules_blocklist = set((
+    # adafruit bundle
+    "adafruit_colorsys",
+
+    # community bundle
+    "at24mac_eeprom",
+    "circuitpython_Candlesticks",
+    "CircuitPython_Color_Picker",
+    "CircuitPython_Equalizer",
+    "CircuitPython_Scales",
+    "circuitPython_Slider",
+    "circuitpython_uboxplot",
+    "P1AM",
+    "p1am_200_helpers",
+))
+
+if sys.version_info >= (3, 11):
+    from tomllib import loads as load_toml
+else:
+    from tomli import loads as load_toml
+
+mpy_cross_path = platformdirs.user_cache_path("circuitpython-build-tools", ensure_exists=True)
+
+def load_pyproject_toml(lib_path: pathlib.Path):
+    try:
+        return load_toml((lib_path / "pyproject.toml") .read_text(encoding="utf-8"))
+    except FileNotFoundError:
+        print(f"No pyproject.toml in {lib_path}")
+        return {}
+
+def get_nested(doc, *args, default=None):
+    for a in args:
+        if doc is None: return default
+        try:
+            doc = doc[a]
+        except (KeyError, IndexError) as e:
+            return default
+    return doc
 
 IGNORE_PY = ["setup.py", "conf.py", "__init__.py"]
-GLOB_PATTERNS = ["*.py", "font5x8.bin"]
+GLOB_PATTERNS = ["*.py", "*.bin"]
+S3_MPY_PREFIX = "https://adafruit-circuit-python.s3.amazonaws.com/bin/mpy-cross"
 
 def version_string(path=None, *, valid_semver=False):
     version = None
@@ -61,72 +128,136 @@ def version_string(path=None, *, valid_semver=False):
             version = commitish
     return version
 
-def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False):
+def mpy_cross(version, quiet=False):
+    circuitpython_tag = version["tag"]
+    name = version["name"]
+    ext = ".exe" * (os.name == "nt")
+    mpy_cross_filename = mpy_cross_path / f"mpy-cross-{name}{ext}"
+    
     if os.path.isfile(mpy_cross_filename):
-        return
+        return mpy_cross_filename
+
+    # Try to pull from S3
+    uname = platform.uname()
+    s3_url = None
+    if uname[0].title() == 'Linux' and uname[4].lower() in ('amd64', 'x86_64'):
+        s3_url = f"{S3_MPY_PREFIX}/linux-amd64/mpy-cross-linux-amd64-{circuitpython_tag}.static"
+    elif uname[0].title() == 'Linux' and uname[4].lower() == 'armv7l':
+        s3_url = f"{S3_MPY_PREFIX}/linux-raspbian/mpy-cross-linux-raspbian-{circuitpython_tag}.static-raspbian"
+    elif uname[0].title() == 'Darwin':
+        s3_url = f"{S3_MPY_PREFIX}/macos-11/mpy-cross-macos-11-{circuitpython_tag}-universal"
+    elif uname[0].title() == "Windows" and uname[4].lower() in ("amd64", "x86_64"):
+        s3_url = f"{S3_MPY_PREFIX}/windows/mpy-cross-windows-{circuitpython_tag}.static.exe"
+    elif not quiet:
+         print(f"Pre-built mpy-cross not available for sysname='{uname[0]}' release='{uname[2]}' machine='{uname[4]}'.")
+
+    if s3_url is not None:
+        if not quiet:
+            print(f"Checking S3 for {s3_url}")
+        try:
+            r = requests.get(s3_url)
+            if r.status_code == 200:
+                with open(mpy_cross_filename, "wb") as f:
+                    f.write(r.content)
+                    # Set the User Execute bit
+                    os.chmod(mpy_cross_filename, os.stat(mpy_cross_filename)[0] | stat.S_IXUSR)
+                    if not quiet:
+                        print("  FOUND")
+                    return mpy_cross_filename
+        except Exception as e:
+            if not quiet:
+                print(f"    exception fetching from S3: {e}")
+        if not quiet:
+            print("  NOT FOUND")
+
     if not quiet:
         title = "Building mpy-cross for circuitpython " + circuitpython_tag
         print()
         print(title)
         print("=" * len(title))
 
-    os.makedirs("build_deps/", exist_ok=True)
-    if not os.path.isdir("build_deps/circuitpython"):
-        clone = subprocess.run("git clone https://github.com/adafruit/circuitpython.git build_deps/circuitpython", shell=True)
-        if clone.returncode != 0:
-            sys.exit(clone.returncode)
+    build_dir = mpy_cross_path / f"build-circuitpython-{circuitpython_tag}"
+    if not os.path.isdir(build_dir):
+        subprocess.check_call(["git", "clone", *git_filter_arg(), "-b", circuitpython_tag, "https://github.com/adafruit/circuitpython.git", build_dir])
 
-    current_dir = os.getcwd()
-    os.chdir("build_deps/circuitpython")
-    make = subprocess.run("git fetch && git checkout {TAG} && git submodule update".format(TAG=circuitpython_tag), shell=True)
-    os.chdir("tools")
-    make = subprocess.run("git submodule update --init .", shell=True)
-    os.chdir("../mpy-cross")
-    make = subprocess.run("make clean && make", shell=True)
-    os.chdir(current_dir)
+    subprocess.check_call(["git", "submodule", "update", "--recursive"], cwd=build_dir)
+    subprocess.check_call([sys.executable, "tools/ci_fetch_deps.py", "mpy-cross"], cwd=build_dir)
+    subprocess.check_call(["make", "clean"], cwd=build_dir / "mpy-cross")
+    subprocess.check_call(["make", f"-j{multiprocessing.cpu_count()}"], cwd=build_dir / "mpy-cross")
 
-    shutil.copy("build_deps/circuitpython/mpy-cross/mpy-cross", mpy_cross_filename)
+    mpy_built = build_dir / f"mpy-cross/build/mpy-cross{ext}"
+    if not os.path.exists(mpy_built):
+        mpy_built = build_dir / f"mpy-cross/mpy-cross{ext}"
 
-    if make.returncode != 0:
-        sys.exit(make.returncode)
+    shutil.copy(mpy_built, mpy_cross_filename)
+    return mpy_cross_filename
 
 def _munge_to_temp(original_path, temp_file, library_version):
-    with open(original_path, "rb") as original_file:
+    with open(original_path, "r", encoding="utf-8") as original_file:
         for line in original_file:
-            if original_path.endswith(".bin"):
-                # this is solely for adafruit_framebuf/examples/font5x8.bin
-                temp_file.write(line)
-            else:
-                line = line.decode("utf-8").strip("\n")
-                if line.startswith("__version__"):
-                    line = line.replace("0.0.0-auto.0", library_version)
-                temp_file.write(line.encode("utf-8") + b"\r\n")
+            line = line.strip("\n")
+            if line.startswith("__version__"):
+                line = line.replace("0.0.0-auto.0", library_version)
+                line = line.replace("0.0.0+auto.0", library_version)
+            print(line, file=temp_file)
     temp_file.flush()
 
-def library(library_path, output_directory, package_folder_prefix,
-            mpy_cross=None, example_bundle=False):
-    py_files = []
-    package_files = []
-    example_files = []
-    total_size = 512
-
+def get_package_info(library_path, package_folder_prefix):
     lib_path = pathlib.Path(library_path)
     parent_idx = len(lib_path.parts)
+    py_files = []
+    package_files = []
+    package_info = {}
     glob_search = []
     for pattern in GLOB_PATTERNS:
         glob_search.extend(list(lib_path.rglob(pattern)))
 
-    for file in glob_search:
-        if file.parts[parent_idx] == "examples":
-            example_files.append(file)
-        else:
-            if not example_bundle:
-                is_package = False
-                for prefix in package_folder_prefix:
-                    if file.parts[parent_idx].startswith(prefix):
-                        is_package = True
+    pyproject_toml = load_pyproject_toml(lib_path)
+    py_modules = get_nested(pyproject_toml, "tool", "setuptools", "py-modules", default=[])
+    packages = get_nested(pyproject_toml, "tool", "setuptools", "packages", default=[])
 
-                if is_package:
+    blocklisted = [name for name in py_modules if name in pyproject_py_modules_blocklist]
+
+    if blocklisted:
+        print(f"{lib_path}/settings.toml:1: {blocklisted[0]} blocklisted: not using metadata from pyproject.toml")
+        py_modules = packages = ()
+
+    example_files = [sub_path for sub_path in (lib_path / "examples").rglob("*")
+            if sub_path.is_file()]
+
+    if packages and py_modules:
+        raise ValueError("Cannot specify both tool.setuptools.py-modules and .packages")
+
+    elif packages:
+        if len(packages) > 1:
+            raise ValueError("Only a single package is supported")
+        package_name = packages[0]
+        #print(f"Using package name from pyproject.toml: {package_name}")
+        package_info["is_package"] = True
+        package_info["module_name"] = package_name
+        package_files = [sub_path for sub_path in (lib_path / package_name).rglob("*")
+                if sub_path.is_file()]
+
+    elif py_modules:
+        if len(py_modules) > 1:
+            raise ValueError("Only a single module is supported")
+        py_module = py_modules[0]
+        #print(f"Using module name from pyproject.toml: {py_module}")
+        package_name = py_module
+        package_info["is_package"] = False
+        package_info["module_name"] = py_module
+        py_files = [lib_path / f"{py_module}.py"]
+
+    else:
+        print(f"{lib_path}: Using legacy autodetection")
+        package_info["is_package"] = False
+        for file in glob_search:
+            if file.parts[parent_idx] != "examples":
+                if len(file.parts) > parent_idx + 1:
+                    for prefix in package_folder_prefix:
+                        if file.parts[parent_idx].startswith(prefix):
+                            package_info["is_package"] = True
+                if package_info["is_package"]:
                     package_files.append(file)
                 else:
                     if file.name in IGNORE_PY:
@@ -135,83 +266,100 @@ def library(library_path, output_directory, package_folder_prefix,
                     if file.parent == lib_path:
                         py_files.append(file)
 
+        if package_files:
+            package_info["module_name"] = package_files[0].relative_to(library_path).parent.name
+        elif py_files:
+            package_info["module_name"] = py_files[0].relative_to(library_path).name[:-3]
+        else:
+            package_info["module_name"] = None
+
     if len(py_files) > 1:
         raise ValueError("Multiple top level py files not allowed. Please put "
                          "them in a package or combine them into a single file.")
 
+    package_info["package_files"] = package_files
+    package_info["py_files"] = py_files
+    package_info["example_files"] = example_files
+
+    try:
+        package_info["version"] = version_string(library_path, valid_semver=True)
+    except ValueError as e:
+        print(library_path + " has version that doesn't follow SemVer (semver.org)")
+        print(e)
+        package_info["version"] = version_string(library_path)
+
+    return package_info
+
+def library(library_path, output_directory, package_folder_prefix,
+            mpy_cross=None, example_bundle=False):
+    lib_path = pathlib.Path(library_path)
+    package_info = get_package_info(library_path, package_folder_prefix)
+    py_package_files = package_info["package_files"] + package_info["py_files"]
+    example_files = package_info["example_files"]
+    module_name = package_info["module_name"]
+
     for fn in example_files:
         base_dir = os.path.join(output_directory.replace("/lib", "/"),
                                 fn.relative_to(library_path).parent)
         if not os.path.isdir(base_dir):
             os.makedirs(base_dir)
-            total_size += 512
 
-    for fn in package_files:
+    for fn in py_package_files:
         base_dir = os.path.join(output_directory,
                                 fn.relative_to(library_path).parent)
         if not os.path.isdir(base_dir):
             os.makedirs(base_dir)
-            total_size += 512
 
-    new_extension = ".py"
-    if mpy_cross:
-        new_extension = ".mpy"
+    library_version = package_info['version']
 
-    try:
-        library_version = version_string(library_path, valid_semver=True)
-    except ValueError as e:
-        print(library_path + " has version that doesn't follow SemVer (semver.org)")
-        print(e)
-        library_version = version_string(library_path)
-
-    for filename in py_files:
-        full_path = os.path.join(library_path, filename)
-        output_file = os.path.join(
-            output_directory,
-            filename.relative_to(library_path).with_suffix(new_extension)
-        )
-        with tempfile.NamedTemporaryFile() as temp_file:
-            _munge_to_temp(full_path, temp_file, library_version)
-
-            if mpy_cross:
-                mpy_success = subprocess.call([
-                    mpy_cross,
-                    "-o", output_file,
-                    "-s", str(filename.relative_to(library_path)),
-                    temp_file.name
-                ])
-                if mpy_success != 0:
-                    raise RuntimeError("mpy-cross failed on", full_path)
+    if not example_bundle:
+        for filename in py_package_files:
+            full_path = os.path.join(library_path, filename)
+            output_file = output_directory / filename.relative_to(library_path)
+            if filename.suffix == ".py":
+                with tempfile.NamedTemporaryFile(delete=False, mode="w+") as temp_file:
+                    temp_file_name = temp_file.name
+                    try:
+                        _munge_to_temp(full_path, temp_file, library_version)
+                        temp_file.close()
+                        if mpy_cross and os.stat(temp_file.name).st_size != 0:
+                            output_file = output_file.with_suffix(".mpy")
+                            mpy_success = subprocess.call([
+                                mpy_cross,
+                                "-o", output_file,
+                                "-s", str(filename.relative_to(library_path)),
+                                temp_file.name
+                            ])
+                            if mpy_success != 0:
+                                raise RuntimeError("mpy-cross failed on", full_path)
+                        else:
+                            shutil.copyfile(full_path, output_file)
+                    finally:
+                        os.remove(temp_file_name)
             else:
-                shutil.copyfile(temp_file.name, output_file)
+                shutil.copyfile(full_path, output_file)
 
-    for filename in package_files:
-        full_path = os.path.join(library_path, filename)
-        with tempfile.NamedTemporaryFile() as temp_file:
-            _munge_to_temp(full_path, temp_file, library_version)
-            if not mpy_cross or os.stat(full_path).st_size == 0:
-                output_file = os.path.join(output_directory,
-                                           filename.relative_to(library_path))
-                shutil.copyfile(temp_file.name, output_file)
-            else:
-                output_file = os.path.join(
-                    output_directory,
-                    filename.relative_to(library_path).with_suffix(new_extension)
-                )
+    requirements_files = lib_path.glob("requirements.txt*")
+    requirements_files = [f for f in requirements_files if f.stat().st_size > 0]
 
-                mpy_success = subprocess.call([
-                    mpy_cross,
-                    "-o", output_file,
-                    "-s", str(filename.relative_to(library_path)),
-                    temp_file.name
-                ])
-                if mpy_success != 0:
-                    raise RuntimeError("mpy-cross failed on", full_path)
+    toml_files = lib_path.glob("pyproject.toml*")
+    toml_files = [f for f in toml_files if f.stat().st_size > 0]
+    requirements_files.extend(toml_files)
+
+    if module_name and requirements_files and not example_bundle:
+        requirements_dir = pathlib.Path(output_directory).parent / "requirements"
+        if not os.path.isdir(requirements_dir):
+            os.makedirs(requirements_dir, exist_ok=True)
+        requirements_subdir = f"{requirements_dir}/{module_name}"
+        if not os.path.isdir(requirements_subdir):
+            os.makedirs(requirements_subdir, exist_ok=True)
+        for filename in requirements_files:
+            full_path = os.path.join(library_path, filename)
+            output_file = os.path.join(requirements_subdir, filename.name)
+            shutil.copyfile(full_path, output_file)
 
     for filename in example_files:
         full_path = os.path.join(library_path, filename)
         output_file = os.path.join(output_directory.replace("/lib", "/"),
                                    filename.relative_to(library_path))
-        with tempfile.NamedTemporaryFile() as temp_file:
-            _munge_to_temp(full_path, temp_file, library_version)
-            shutil.copyfile(temp_file.name, output_file)
+        shutil.copyfile(full_path, output_file)

circuitpython_build_tools/scripts/build_bundles.py

@@ -25,7 +25,7 @@
 import json
 import os
 import os.path
-import shlex
+import re
 import shutil
 import subprocess
 import sys
@@ -36,7 +36,28 @@ import click
 from circuitpython_build_tools import build
 from circuitpython_build_tools import target_versions
 
-import pkg_resources
+if sys.version_info < (3, 8):
+    import importlib_metadata
+else:
+    import importlib.metadata as importlib_metadata
+
+BLINKA_LIBRARIES = [
+    "adafruit-blinka",
+    "adafruit-blinka-bleio",
+    "adafruit-blinka-displayio",
+    "adafruit-blinka-pyportal",
+    "adafruit-python-extended-bus",
+    "numpy",
+    "pillow",
+    "pyasn1",
+    "pyserial",
+    "scipy",
+    "spidev",
+]
+
+def normalize_dist_name(name: str) -> str:
+    """Return a normalized pip name"""
+    return name.lower().replace("_", "-")
 
 def add_file(bundle, src_file, zip_name):
     bundle.write(src_file, zip_name)
@@ -47,9 +68,90 @@ def add_file(bundle, src_file, zip_name):
     print(zip_name, file_size, file_sector_size)
     return file_sector_size
 
+def get_module_name(library_path, remote_name):
+    """Figure out the module or package name and return it"""
+    repo = subprocess.run(f'git remote get-url {remote_name}', shell=True, stdout=subprocess.PIPE, cwd=library_path)
+    repo = repo.stdout.decode("utf-8", errors="ignore").strip().lower()
+    if repo[-4:] == ".git":
+        repo = repo[:-4]
+    module_name = normalize_dist_name(repo.split("/")[-1])
+
+    # circuitpython org repos are deployed to pypi without "org" in the pypi name
+    module_name = re.sub(r"^circuitpython-org-", "circuitpython-", module_name)
+    return module_name, repo
+
+def get_bundle_requirements(directory, package_list):
+    """
+    Open the requirements.txt if it exists
+    Remove anything that shouldn't be a requirement like Adafruit_Blinka
+    Return the list
+    """
+    
+    pypi_reqs = set()   # For multiple bundle dependency
+    dependencies = set()   # For intra-bundle dependency
+    
+    path = directory + "/requirements.txt"
+    if os.path.exists(path):
+        with open(path, "r") as file:
+            requirements = file.read()
+            file.close()
+            for line in requirements.split("\n"):
+                line = line.lower().strip()
+                if line.startswith("#") or line == "":
+                    # skip comments
+                    pass
+                else:
+                    # Remove any pip version and platform specifiers
+                    original_name = re.split("[<>=~[;]", line)[0].strip()
+                    # Normalize to match the indexes in package_list
+                    line = normalize_dist_name(original_name)
+                    if line in package_list:
+                        dependencies.add(package_list[line]["module_name"])
+                    elif line not in BLINKA_LIBRARIES:
+                        # add with the exact spelling from requirements.txt
+                        pypi_reqs.add(original_name)
+    return sorted(dependencies), sorted(pypi_reqs)
+
+def build_bundle_json(libs, bundle_version, output_filename, package_folder_prefix, remote_name="origin"):
+    """
+    Generate a JSON file of all the libraries in libs
+    """
+    packages = {}
+    # TODO simplify this 2-step process
+    # It mostly exists so that get_bundle_requirements has a way to look up
+    # "pypi name to bundle name" via `package_list[pypi_name]["module_name"]`
+    # otherwise it's just shuffling info around
+    for library_path in libs:
+        package = {}
+        package_info = build.get_package_info(library_path, package_folder_prefix)
+        module_name, repo = get_module_name(library_path, remote_name)
+        if package_info["module_name"] is not None:
+            package["module_name"] = package_info["module_name"]
+            package["pypi_name"] = module_name
+            package["repo"] = repo
+            package["is_folder"] = package_info["is_package"]
+            package["version"] = package_info["version"]
+            package["path"] = "lib/" + package_info["module_name"]
+            package["library_path"] = library_path
+            packages[module_name] = package
+
+    library_submodules = {}
+    for package in packages.values():
+        library = {}
+        library["package"] = package["is_folder"]
+        library["pypi_name"] = package["pypi_name"]
+        library["version"] = package["version"]
+        library["repo"] = package["repo"]
+        library["path"] = package["path"]
+        library["dependencies"], library["external_dependencies"] = get_bundle_requirements(package["library_path"], packages)
+        library_submodules[package["module_name"]] = library
+
+    out_file = open(output_filename, "w")
+    json.dump(library_submodules, out_file, sort_keys=True)
+    out_file.close()
 
 def build_bundle(libs, bundle_version, output_filename, package_folder_prefix,
-        build_tools_version="devel", mpy_cross=None, example_bundle=False):
+        build_tools_version="devel", mpy_cross=None, example_bundle=False, remote_name="origin"):
     build_dir = "build-" + os.path.basename(output_filename)
     top_folder = os.path.basename(output_filename).replace(".zip", "")
     build_lib_dir = os.path.join(build_dir, top_folder, "lib")
@@ -81,7 +183,7 @@ def build_bundle(libs, bundle_version, output_filename, package_folder_prefix,
     if multiple_libs:
         with open(os.path.join(build_dir, top_folder, "VERSIONS.txt"), "w") as f:
             f.write(bundle_version + "\r\n")
-            versions = subprocess.run('git submodule --quiet foreach \"git remote get-url origin && git describe --tags\"', shell=True, stdout=subprocess.PIPE, cwd=os.path.commonpath(libs))
+            versions = subprocess.run(f'git submodule --quiet foreach \"git remote get-url {remote_name} && git describe --tags\"', shell=True, stdout=subprocess.PIPE, cwd=os.path.commonpath(libs))
             if versions.returncode != 0:
                 print("Failed to generate versions file. Its likely a library hasn't been "
                       "released yet.")
@@ -107,7 +209,7 @@ def build_bundle(libs, bundle_version, output_filename, package_folder_prefix,
     print()
     print("Zipping")
 
-    with zipfile.ZipFile(output_filename, 'w') as bundle:
+    with zipfile.ZipFile(output_filename, 'w', compression=zipfile.ZIP_DEFLATED) as bundle:
         build_metadata = {"build-tools-version": build_tools_version}
         bundle.comment = json.dumps(build_metadata).encode("utf-8")
         if multiple_libs:
@@ -132,13 +234,17 @@ def _find_libraries(current_path, depth):
             subdirectories.extend(_find_libraries(path, depth - 1))
     return subdirectories
 
+all_modules = ["py", "mpy", "example", "json"]
 @click.command()
 @click.option('--filename_prefix', required=True, help="Filename prefix for the output zip files.")
 @click.option('--output_directory', default="bundles", help="Output location for the zip files.")
 @click.option('--library_location', required=True, help="Location of libraries to bundle.")
 @click.option('--library_depth', default=0, help="Depth of library folders. This is useful when multiple libraries are bundled together but are initially in separate subfolders.")
 @click.option('--package_folder_prefix', default="adafruit_", help="Prefix string used to determine package folders to bundle.")
-def build_bundles(filename_prefix, output_directory, library_location, library_depth, package_folder_prefix):
+@click.option('--remote_name', default="origin", help="Git remote name to use during building")
+@click.option('--ignore', "-i", multiple=True, type=click.Choice(all_modules), help="Bundles to ignore building")
+@click.option('--only', "-o", multiple=True, type=click.Choice(all_modules), help="Bundles to build building")
+def build_bundles(filename_prefix, output_directory, library_location, library_depth, package_folder_prefix, remote_name, ignore, only):
     os.makedirs(output_directory, exist_ok=True)
 
     package_folder_prefix = package_folder_prefix.split(", ")
@@ -147,10 +253,10 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d
 
     libs = _find_libraries(os.path.abspath(library_location), library_depth)
 
-    pkg = pkg_resources.get_distribution("circuitpython-build-tools")
-    build_tools_version = "devel"
-    if pkg:
-        build_tools_version = pkg.version
+    try:
+        build_tools_version = importlib_metadata.version("circuitpython-build-tools")
+    except importlib_metadata.PackageNotFoundError:
+        build_tools_version = "devel"
 
     build_tools_fn = "z-build_tools_version-{}.ignore".format(
         build_tools_version)
@@ -158,33 +264,41 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d
     with open(build_tools_fn, "w") as f:
         f.write(build_tools_version)
 
-    # Build raw source .py bundle
-    zip_filename = os.path.join(output_directory,
-        filename_prefix + '-py-{VERSION}.zip'.format(
-            VERSION=bundle_version))
-    build_bundle(libs, bundle_version, zip_filename, package_folder_prefix,
-                 build_tools_version=build_tools_version)
+    if ignore and only:
+        raise SystemExit("Only specify one of --ignore / --only")
+    if only:
+        ignore = set(all_modules) - set(only)
 
-    # Build .mpy bundle(s)
-    os.makedirs("build_deps", exist_ok=True)
-    for version in target_versions.VERSIONS:
-        # Use prebuilt mpy-cross on Travis, otherwise build our own.
-        if "TRAVIS" in os.environ:
-            mpy_cross = pkg_resources.resource_filename(
-                target_versions.__name__, "data/mpy-cross-" + version["name"])
-        else:
-            mpy_cross = "build_deps/mpy-cross-" + version["name"]
-            build.mpy_cross(mpy_cross, version["tag"])
+    # Build raw source .py bundle
+    if "py" not in ignore:
         zip_filename = os.path.join(output_directory,
-            filename_prefix + '-{TAG}-mpy-{VERSION}.zip'.format(
-                TAG=version["name"],
+            filename_prefix + '-py-{VERSION}.zip'.format(
                 VERSION=bundle_version))
         build_bundle(libs, bundle_version, zip_filename, package_folder_prefix,
-                     mpy_cross=mpy_cross, build_tools_version=build_tools_version)
+                    build_tools_version=build_tools_version, remote_name=remote_name)
+
+    # Build .mpy bundle(s)
+    if "mpy" not in ignore:
+        for version in target_versions.VERSIONS:
+            mpy_cross = build.mpy_cross(version)
+            zip_filename = os.path.join(output_directory,
+                filename_prefix + '-{TAG}-mpy-{VERSION}.zip'.format(
+                    TAG=version["name"],
+                    VERSION=bundle_version))
+            build_bundle(libs, bundle_version, zip_filename, package_folder_prefix,
+                        mpy_cross=mpy_cross, build_tools_version=build_tools_version, remote_name=remote_name)
 
     # Build example bundle
-    zip_filename = os.path.join(output_directory,
-        filename_prefix + '-examples-{VERSION}.zip'.format(
-            VERSION=bundle_version))
-    build_bundle(libs, bundle_version, zip_filename, package_folder_prefix,
-                 build_tools_version=build_tools_version, example_bundle=True)
+    if "example" not in ignore:
+        zip_filename = os.path.join(output_directory,
+            filename_prefix + '-examples-{VERSION}.zip'.format(
+                VERSION=bundle_version))
+        build_bundle(libs, bundle_version, zip_filename, package_folder_prefix,
+                    build_tools_version=build_tools_version, example_bundle=True, remote_name=remote_name)
+
+    # Build Bundle JSON
+    if "json" not in ignore:
+        json_filename = os.path.join(output_directory,
+            filename_prefix + '-{VERSION}.json'.format(
+                VERSION=bundle_version))
+        build_bundle_json(libs, bundle_version, json_filename, package_folder_prefix, remote_name=remote_name)

circuitpython_build_tools/scripts/build_mpy_cross.py

@@ -28,9 +28,14 @@ from circuitpython_build_tools import target_versions
 import os
 import sys
 
+import click
+
+@click.command
+@click.argument("versions")
+def main(versions):
+    print(versions)
+    for version in [v for v in target_versions.VERSIONS if v['name'] in versions]:
+        print(f"{version['name']}: {build.mpy_cross(version)}")
+
 if __name__ == "__main__":
-    output_directory = sys.argv[1]
-    os.makedirs(output_directory, exist_ok=True)
-    for version in target_versions.VERSIONS:
-        mpy_cross = output_directory + "/mpy-cross-" + version["name"]
-        build.mpy_cross(mpy_cross, version["tag"])
+    main()

circuitpython_build_tools/scripts/circuitpython_mpy_cross.py

@@ -0,0 +1,21 @@
+import subprocess
+
+import click
+
+from ..target_versions import VERSIONS
+from ..build import mpy_cross
+
+@click.command(context_settings={"ignore_unknown_options": True})
+@click.option("--circuitpython-version", type=click.Choice([version["name"] for version in VERSIONS]))
+@click.option("--quiet/--no-quiet", "quiet", type=bool, default=True)
+@click.argument("mpy-cross-args", nargs=-1, required=True)
+def main(circuitpython_version, quiet, mpy_cross_args):
+    version_info, = [v for v in VERSIONS if v["name"] == circuitpython_version]
+    mpy_cross_exe = str(mpy_cross(version_info, quiet))
+    try:
+        subprocess.check_call([mpy_cross_exe, *mpy_cross_args])
+    except subprocess.CalledProcessError as e:
+        raise SystemExit(e.returncode)
+
+if __name__ == '__main__':
+    main()

circuitpython_build_tools/target_versions.py

@@ -25,5 +25,6 @@
 # The tag specifies which version of CircuitPython to use for mpy-cross.
 # The name is used when constructing the zip file names.
 VERSIONS = [
-    {"tag": "6.1.0", "name": "6.x"},
+    {"tag": "8.2.0", "name": "8.x"},
+    {"tag": "9.0.0-alpha.2", "name": "9.x"},
 ]

requirements.txt

@@ -1,3 +1,6 @@
 Click
+requests
 semver
 wheel
+tomli; python_version < "3.11"
+platformdirs

setup.py

@@ -13,10 +13,11 @@ setup(name='circuitpython-build-tools',
                 'circuitpython_build_tools.scripts'],
       package_data={'circuitpython_build_tools': ['data/mpy-cross-*']},
       zip_safe=False,
-      python_requires='>=3.4',
-      install_requires=['Click', 'semver'],
+      python_requires='>=3.10',
+      install_requires=['Click', 'requests', 'semver', 'tomli; python_version < "3.11"', 'platformdirs'],
       entry_points='''
         [console_scripts]
         circuitpython-build-bundles=circuitpython_build_tools.scripts.build_bundles:build_bundles
+        circuitpython-mpy-cross=circuitpython_build_tools.scripts.circuitpython_mpy_cross:main
       '''
       )