update tag for 7.x

Convert from Travis to Github Actions

.github/workflows/build.yml

@@ -0,0 +1,52 @@
+# SPDX-FileCopyrightText: 2017 Scott Shawcroft, written for Adafruit Industries
+# SPDX-FileCopyrightText: 2021 James Carr
+#
+# SPDX-License-Identifier: MIT
+
+name: Build CI
+
+on: [pull_request, push]
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Dump GitHub context
+      env:
+        GITHUB_CONTEXT: ${{ toJson(github) }}
+      run: echo "$GITHUB_CONTEXT"
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Versions
+      run: |
+        python3 --version
+    - name: Checkout Current Repo
+      uses: actions/checkout@v2
+    - name: Install requirements
+      run: |
+        sudo apt-get update
+        sudo apt-get install libudev-dev libusb-1.0
+        sudo apt-get install -y gettext
+        pip install -r requirements.txt
+    - name: Library version
+      run: git describe --dirty --always --tags
+    - name: Install package locally
+      run: pip install -e .
+    - name: Test building single package
+      run: |
+        git clone https://github.com/adafruit/Adafruit_CircuitPython_FeatherWing.git
+        cd Adafruit_CircuitPython_FeatherWing
+        circuitpython-build-bundles --filename_prefix test-single --library_location .
+    - name: Test building bundle
+      run: |
+        # Use the community bundle because it's smaller and faster
+        git clone --recurse-submodules https://github.com/adafruit/CircuitPython_Community_Bundle.git
+        cd CircuitPython_Community_Bundle
+        circuitpython-build-bundles --filename_prefix test-bundle --library_location libraries --library_depth 2
+    - name: Build Python package
+      run: |
+        pip install --upgrade setuptools wheel twine readme_renderer testresources
+        python setup.py sdist
+        twine check dist/*

.github/workflows/release.yml

@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: 2017 Scott Shawcroft, written for Adafruit Industries
+# SPDX-FileCopyrightText: 2021 James Carr
+#
+# SPDX-License-Identifier: MIT
+
+name: Release Actions
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  upload-pypi:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: '3.7'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine
+    - name: Build and publish
+      env:
+        TWINE_USERNAME: ${{ secrets.pypi_username }}
+        TWINE_PASSWORD: ${{ secrets.pypi_password }}
+      run: |
+        python setup.py sdist
+        twine upload dist/*

.gitignore

@@ -8,3 +8,5 @@ circuitpython_build_tools/data/
 .eggs
 version.py
 .env/*
+.DS_Store
+.idea/*

.travis.yml

@@ -1,50 +0,0 @@
-dist: xenial
-language: python
-python:
-- '3.6'
-
-stages:
-  - name: Tests
-    if: type = pull_request
-  - name: deploy
-    if: tag IS present
-
-jobs:
-  include:
-    - stage: Tests
-      name: "Test CircuitPython Bundle"
-      python: "3.6"
-      script:
-        - echo "Building mpy-cross" && echo "travis_fold:start:mpy-cross"
-        - python3 -u -m circuitpython_build_tools.scripts.build_mpy_cross circuitpython_build_tools/data/
-        - echo "travis_fold:end:mpy-cross"
-        - pip install -e .
-        - echo "Cloning Adafruit_CircuitPython_Bundle" && echo "travis_fold:start:clone"
-        - git clone --recurse-submodules https://github.com/adafruit/Adafruit_CircuitPython_Bundle.git
-        - echo "travis_fold:end:clone"
-        - cd Adafruit_CircuitPython_Bundle
-        - circuitpython-build-bundles --filename_prefix test-bundle --library_location libraries --library_depth 2
-
-    - stage: Tests
-      name: "Test Single Library Bundle"
-      script:
-        - echo "Building mpy-cross" && echo "travis_fold:start:mpy-cross"
-        - python3 -u -m circuitpython_build_tools.scripts.build_mpy_cross circuitpython_build_tools/data/
-        - echo "travis_fold:end:mpy-cross"
-        - pip install -e .
-        - git clone https://github.com/adafruit/Adafruit_CircuitPython_FeatherWing.git
-        - cd Adafruit_CircuitPython_FeatherWing
-        - circuitpython-build-bundles --filename_prefix test-single --library_location .
-
-    - stage: deploy
-      script:
-        - python3 -u -m circuitpython_build_tools.scripts.build_mpy_cross circuitpython_build_tools/data/
-      deploy:
-        provider: pypi
-        user: "adafruit-travis"
-        password:
-          secure: tCjXwD8YeitG0HZLxW1D1QlVv4Xbj8mfRoqW0CM9ikPp3KY1PCz6Axj0PiOcyVwKdnxcUQ0EGRl16wEqwkObrmo9MboYrPuPqN00ULmyCQCRvJa2abIN6jDoLtBuf6bcze88t0XY2LdMOcj2Udv5Iijgf95zUgE+Z6BqT9Rgche78JEOeANJ7BlAJ6nRCA4whDdG7J9s7SmFtIjKWtMxig2J3X+Qy0bZ+Armtfp9/CRvjLJ8juDrcCBSysWLnAYLS4u8e/rbSTh8YwFeoeJ1pp9qSmME5NuwScY18QmfESNSqz8wVVXtAFKdoMOCoN+/CodTxp9aB0QsXX6yOYg74ahDIaci239wgnuUqxSaeLxeSwWkkVCXWdQVuP4vgq3GZwm2yNOQ1ZjfFbXF156yv0uSVw5nuaxv0YblQTinJtL4x9hwOdPDJio3b6UT3H1ue9l1qK0LT2OSkzDgn12WmTnTfRUH3BkU6onsYsdP33PK1YhepeQnfbT1P3ikrRHIwGYb7XqcjOtJh413kid6YezCXRqccl8kAxegnqX+cQG7K9ilpZtWaVYLu4RRBJ37H4vpuOb3SV686Y62sWPUXEbI3MR1OxU+RrRr/9DCH1EFXnlYT9LF986wXFJtWuSc+pbXuxY7qduai0hn5Pft6XH31exyiOwAHBIFeYebnVM=
-        skip_cleanup: true
-        on:
-          tags: true
-

circuitpython_build_tools/build.py

@@ -4,6 +4,7 @@
 #
 # Copyright (c) 2016 Scott Shawcroft for Adafruit Industries
 #               2018, 2019 Michael Schroeder
+#               2021 James Carr
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -26,14 +27,17 @@
 import os
 import os.path
 import pathlib
+import requests
 import semver
 import shutil
+import stat
 import sys
 import subprocess
 import tempfile
 
 IGNORE_PY = ["setup.py", "conf.py", "__init__.py"]
 GLOB_PATTERNS = ["*.py", "font5x8.bin"]
+S3_MPY_PREFIX = "https://adafruit-circuit-python.s3.amazonaws.com/bin/mpy-cross/"
 
 def version_string(path=None, *, valid_semver=False):
     version = None
@@ -64,6 +68,38 @@ def version_string(path=None, *, valid_semver=False):
 def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False):
     if os.path.isfile(mpy_cross_filename):
         return
+
+    # Try to pull from S3
+    uname = os.uname()
+    s3_url = None
+    if uname[0] == 'Linux' and uname[4] in ('amd64', 'x86_64'):
+        s3_url = f"{S3_MPY_PREFIX}mpy-cross.static-amd64-linux-{circuitpython_tag}"
+    elif uname[0] == 'Linux' and uname[4] == 'armv7l':
+        s3_url = f"{S3_MPY_PREFIX}mpy-cross.static-raspbian-{circuitpython_tag}"
+    elif uname[0] == 'Darwin' and uname[4] == 'x86_64':
+        s3_url = f"{S3_MPY_PREFIX}mpy-cross-macos-catalina-{circuitpython_tag}"
+    elif not quiet:
+         print(f"Pre-built mpy-cross not available for sysname='{uname[0]}' release='{uname[2]}' machine='{uname[4]}'.")
+
+    if s3_url is not None:
+        if not quiet:
+            print(f"Checking S3 for {s3_url}")
+        try:
+            r = requests.get(s3_url)
+            if r.status_code == 200:
+                with open(mpy_cross_filename, "wb") as f:
+                    f.write(r.content)
+                    # Set the User Execute bit
+                    os.chmod(mpy_cross_filename, os.stat(mpy_cross_filename)[0] | stat.S_IXUSR)
+                    if not quiet:
+                        print("  FOUND")
+                    return
+        except Exception as e:
+            if not quiet:
+                print(f"    exception fetching from S3: {e}")
+        if not quiet:
+            print("  NOT FOUND")
+
     if not quiet:
         title = "Building mpy-cross for circuitpython " + circuitpython_tag
         print()
@@ -103,6 +139,46 @@ def _munge_to_temp(original_path, temp_file, library_version):
                 temp_file.write(line.encode("utf-8") + b"\r\n")
     temp_file.flush()
 
+def get_package_info(library_path, package_folder_prefix):
+    lib_path = pathlib.Path(library_path)
+    parent_idx = len(lib_path.parts)
+    py_files = []
+    package_files = []
+    package_info = {}
+    glob_search = []
+    for pattern in GLOB_PATTERNS:
+        glob_search.extend(list(lib_path.rglob(pattern)))
+
+    package_info["is_package"] = False
+    for file in glob_search:
+        if file.parts[parent_idx] != "examples":
+            if len(file.parts) > parent_idx + 1:
+                for prefix in package_folder_prefix:
+                    if file.parts[parent_idx].startswith(prefix):
+                        package_info["is_package"] = True
+            if package_info["is_package"]:
+                package_files.append(file)
+            else:
+                if file.name in IGNORE_PY:
+                    #print("Ignoring:", file.resolve())
+                    continue
+                if file.parent == lib_path:
+                    py_files.append(file)
+
+    if package_files:
+        package_info["module_name"] = package_files[0].relative_to(library_path).parent.name
+    elif py_files:
+        package_info["module_name"] = py_files[0].relative_to(library_path).name[:-3]
+    else:
+        package_info["module_name"] = None
+ 
+    try:
+        package_info["version"] = version_string(library_path, valid_semver=True)
+    except ValueError as e:
+        package_info["version"] = version_string(library_path)
+
+    return package_info
+
 def library(library_path, output_directory, package_folder_prefix,
             mpy_cross=None, example_bundle=False):
     py_files = []
@@ -122,9 +198,10 @@ def library(library_path, output_directory, package_folder_prefix,
         else:
             if not example_bundle:
                 is_package = False
-                for prefix in package_folder_prefix:
-                    if file.parts[parent_idx].startswith(prefix):
-                        is_package = True
+                if len(file.parts) > parent_idx + 1:
+                    for prefix in package_folder_prefix:
+                        if file.parts[parent_idx].startswith(prefix):
+                            is_package = True
 
                 if is_package:
                     package_files.append(file)
@@ -139,6 +216,13 @@ def library(library_path, output_directory, package_folder_prefix,
         raise ValueError("Multiple top level py files not allowed. Please put "
                          "them in a package or combine them into a single file.")
 
+    if package_files:
+        module_name = package_files[0].relative_to(library_path).parent.name
+    elif py_files:
+        module_name = py_files[0].relative_to(library_path).name[:-3]
+    else:
+        module_name = None
+
     for fn in example_files:
         base_dir = os.path.join(output_directory.replace("/lib", "/"),
                                 fn.relative_to(library_path).parent)
@@ -208,6 +292,22 @@ def library(library_path, output_directory, package_folder_prefix,
                 if mpy_success != 0:
                     raise RuntimeError("mpy-cross failed on", full_path)
 
+    requirements_files = lib_path.glob("requirements.txt*")
+    requirements_files = [f for f in requirements_files if f.stat().st_size > 0]
+    if module_name and requirements_files and not example_bundle:
+        requirements_dir = pathlib.Path(output_directory).parent / "requirements"
+        if not os.path.isdir(requirements_dir):
+            os.makedirs(requirements_dir, exist_ok=True)
+            total_size += 512
+        requirements_subdir = f"{requirements_dir}/{module_name}"
+        if not os.path.isdir(requirements_subdir):
+            os.makedirs(requirements_subdir, exist_ok=True)
+            total_size += 512
+        for filename in requirements_files:
+            full_path = os.path.join(library_path, filename)
+            output_file = os.path.join(requirements_subdir, "requirements.txt")
+            shutil.copyfile(full_path, output_file)
+
     for filename in example_files:
         full_path = os.path.join(library_path, filename)
         output_file = os.path.join(output_directory.replace("/lib", "/"),

circuitpython_build_tools/scripts/build_bundles.py

@@ -25,6 +25,7 @@
 import json
 import os
 import os.path
+import re
 import shlex
 import shutil
 import subprocess
@@ -38,6 +39,15 @@ from circuitpython_build_tools import target_versions
 
 import pkg_resources
 
+BLINKA_LIBRARIES = [
+    "adafruit-blinka",
+    "adafruit-blinka-bleio",
+    "adafruit-blinka-displayio",
+    "adafruit-blinka-pyportal",
+    "adafruit-python-extended-bus",
+    "pyserial",
+]
+
 def add_file(bundle, src_file, zip_name):
     bundle.write(src_file, zip_name)
     file_size = os.stat(src_file).st_size
@@ -47,6 +57,80 @@ def add_file(bundle, src_file, zip_name):
     print(zip_name, file_size, file_sector_size)
     return file_sector_size
 
+def get_module_name(library_path):
+    """Figure out the module or package name and return it"""
+    repo = subprocess.run('git remote get-url origin', shell=True, stdout=subprocess.PIPE, cwd=library_path)
+    repo = repo.stdout.decode("utf-8", errors="ignore").strip().lower()
+    if repo[-4:] == ".git":
+        repo = repo[:-4]
+    module_name = repo.split("/")[-1].replace("_", "-")
+
+    # circuitpython org repos are deployed to pypi without "org" in the pypi name
+    module_name = re.sub(r"^circuitpython-org-", "circuitpython-", module_name)
+    return module_name, repo
+
+def get_bundle_requirements(directory, package_list):
+    """
+    Open the requirements.txt if it exists
+    Remove anything that shouldn't be a requirement like Adafruit_Blinka
+    Return the list
+    """
+    
+    pypi_reqs = []   # For multiple bundle dependency
+    dependencies = []   # For intra-bundle dependency
+    
+    path = directory + "/requirements.txt"
+    if os.path.exists(path):
+        with open(path, "r") as file:
+            requirements = file.read()
+            file.close()
+            for line in requirements.split("\n"):
+                line = line.lower().strip()
+                if line.startswith("#") or line == "":
+                    # skip comments
+                    pass
+                else:
+                    if any(operators in line for operators in [">", "<", "="]):
+                        # Remove everything after any pip style version specifiers
+                        line = re.split("[<|>|=|]", line)[0]
+                    if line not in dependencies and line in package_list:
+                        dependencies.append(package_list[line]["module_name"])
+                    elif line not in pypi_reqs and line not in BLINKA_LIBRARIES:
+                        pypi_reqs.append(line)
+    return dependencies, pypi_reqs
+
+def build_bundle_json(libs, bundle_version, output_filename, package_folder_prefix):
+    """
+    Generate a JSON file of all the libraries in libs
+    """
+    packages = {}
+    for library_path in libs:
+        package = {}
+        package_info = build.get_package_info(library_path, package_folder_prefix)
+        module_name, repo = get_module_name(library_path)
+        if package_info["module_name"] is not None:
+            package["module_name"] = package_info["module_name"]
+            package["pypi_name"] = module_name
+            package["repo"] = repo
+            package["is_folder"] = package_info["is_package"]
+            package["version"] = package_info["version"]
+            package["path"] = "lib/" + package_info["module_name"]
+            package["library_path"] = library_path
+            packages[module_name] = package
+
+    library_submodules = {}
+    for id in packages:
+        library = {}
+        library["package"] = packages[id]["is_folder"]
+        library["pypi_name"] = packages[id]["pypi_name"]
+        library["version"] = packages[id]["version"]
+        library["repo"] = packages[id]["repo"]
+        library["path"] = packages[id]["path"]
+        library["dependencies"], library["external_dependencies"] = get_bundle_requirements(packages[id]["library_path"], packages)
+        library_submodules[packages[id]["module_name"]] = library
+    out_file = open(output_filename, "w")
+    json.dump(library_submodules, out_file)
+    out_file.close()
 
 def build_bundle(libs, bundle_version, output_filename, package_folder_prefix,
         build_tools_version="devel", mpy_cross=None, example_bundle=False):
@@ -188,3 +272,9 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d
             VERSION=bundle_version))
     build_bundle(libs, bundle_version, zip_filename, package_folder_prefix,
                  build_tools_version=build_tools_version, example_bundle=True)
+
+    # Build Bundle JSON
+    json_filename = os.path.join(output_directory,
+        filename_prefix + '-{VERSION}.json'.format(
+            VERSION=bundle_version))
+    build_bundle_json(libs, bundle_version, json_filename, package_folder_prefix)

circuitpython_build_tools/target_versions.py

@@ -25,5 +25,6 @@
 # The tag specifies which version of CircuitPython to use for mpy-cross.
 # The name is used when constructing the zip file names.
 VERSIONS = [
-    {"tag": "6.1.0", "name": "6.x"},
+    {"tag": "6.3.0", "name": "6.x"},
+    {"tag": "7.0.0", "name": "7.x"},
 ]

requirements.txt

@@ -1,3 +1,4 @@
 Click
+requests
 semver
 wheel

setup.py

@@ -13,8 +13,8 @@ setup(name='circuitpython-build-tools',
                 'circuitpython_build_tools.scripts'],
       package_data={'circuitpython_build_tools': ['data/mpy-cross-*']},
       zip_safe=False,
-      python_requires='>=3.4',
-      install_requires=['Click', 'semver'],
+      python_requires='>=3.7',
+      install_requires=['Click', 'requests', 'semver'],
       entry_points='''
         [console_scripts]
         circuitpython-build-bundles=circuitpython_build_tools.scripts.build_bundles:build_bundles