Merge pull request #253 from FoamyGuy/autofile_outside_root_subimports

fix subimport finding for auto-files that are not in the root.

.github/workflows/build.yml

@@ -16,11 +16,11 @@ jobs:
       run: echo "$GITHUB_CONTEXT"
     - name: Translate Repo Name For Build Tools filename_prefix
       id: repo-name
-      run: echo ::set-output name=repo-name::circup
-    - name: Set up Python 3.7
+      run: echo "repo-name=circup" >> $GITHUB_OUTPUT
+    - name: Set up Python 3.11
       uses: actions/setup-python@v1
       with:
-        python-version: 3.7
+        python-version: 3.11
     - name: Pip install Sphinx & pre-commit
       run: |
         pip install --force-reinstall Sphinx sphinx-rtd-theme pre-commit

.github/workflows/release.yml

@@ -1,5 +1,6 @@
-# SPDX-FileCopyrightText: 2019 Nicholas Tollervey, written for Adafruit Industries
-
+# SPDX-FileCopyrightText: 2017 Scott Shawcroft, written for Adafruit Industries
+# SPDX-FileCopyrightText: 2021 James Carr
+#
 # SPDX-License-Identifier: MIT
 
 name: Release Actions
@@ -14,26 +15,20 @@ jobs:
     steps:
     - uses: actions/checkout@v4
       with:
-        show-progress: false
-    - name: Check For setup.py
-      id: need-pypi
-      run: |
-        echo ::set-output name=setup-py::$( find . -wholename './setup.py' )
+        filter: 'blob:none'
+        depth: 0
     - name: Set up Python
-      if: contains(steps.need-pypi.outputs.setup-py, 'setup.py')
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.x'
+        python-version: '3.11'
     - name: Install dependencies
-      if: contains(steps.need-pypi.outputs.setup-py, 'setup.py')
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools wheel twine
+        pip install build setuptools wheel twine
     - name: Build and publish
-      if: contains(steps.need-pypi.outputs.setup-py, 'setup.py')
       env:
         TWINE_USERNAME: ${{ secrets.pypi_username }}
         TWINE_PASSWORD: ${{ secrets.pypi_password }}
       run: |
-        python setup.py sdist
+        python -m build
         twine upload dist/*

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
     -   id: black
         exclude: "^tests/bad_python.py$"
 -   repo: https://github.com/pycqa/pylint
-    rev: v2.15.5
+    rev: v3.1.0
     hooks:
     -   id: pylint
         name: lint (examples)

.pylintrc

@@ -246,7 +246,7 @@ ignore-docstrings=yes
 ignore-imports=yes
 
 # Minimum lines number of a similarity.
-min-similarity-lines=4
+min-similarity-lines=8
 
 
 [BASIC]

CONTRIBUTING.rst

@@ -25,15 +25,24 @@ Developer Setup
 
 .. note::
 
-    Please try to use Python 3.6+ while developing CircUp. This is so we can
+    Please try to use Python 3.9+ while developing Circup. This is so we can
     use the
     `Black code formatter <https://black.readthedocs.io/en/stable/index.html>`_
-    (which only works with Python 3.6+).
+    and so that we're supporting versions which still receive security updates.
+
 
 Clone the repository and from the root of the project,
-install the requirements::
 
-    pip install -e ".[dev]"
+
+If you'd like you can setup a virtual environment and activate it.::
+
+    python3 -m venv .env
+    source .env/bin/activate
+
+install the development requirements::
+
+    pip install -r optional_requirements.txt
+
 
 Run the test suite::
 
@@ -91,7 +100,7 @@ subsequently used to facilitate the various commands the tool makes available.
 
 These commands are defined at the very end of the ``circup.py`` code.
 
-Unit tests can be found in the ``tests`` directory. CircUp uses
+Unit tests can be found in the ``tests`` directory. Circup uses
 `pytest <http://www.pytest.org/en/latest/>`_ style testing conventions. Test
 functions should include a comment to describe its *intention*. We currently
 have 100% unit test coverage for all the core functionality (excluding
@@ -115,7 +124,7 @@ available options to help you work with the code base.
 Before submitting a PR, please remember to ``pre-commit run --all-files``.
 But if  you forget the CI process in Github will run it for you. ;-)
 
-CircUp uses the `Click <https://click.palletsprojects.com/en/7.x/>`_ module to
+Circup uses the `Click <https://click.palletsprojects.com>`_ module to
 run command-line interaction. The
 `AppDirs <https://pypi.org/project/appdirs/>`_ module is used to determine
 where to store user-specific assets created by the tool in such a way that

README.rst

@@ -1,5 +1,5 @@
 
-CircUp
+Circup
 ======
 
 .. image:: https://readthedocs.org/projects/circup/badge/?version=latest
@@ -28,7 +28,7 @@ A tool to manage and update libraries (modules) on a CircuitPython device.
 Installation
 ------------
 
-Circup requires Python 3.5 or higher.
+Circup requires Python 3.9 or higher.
 
 In a `virtualenv <https://virtualenv.pypa.io/en/latest/>`_,
 ``pip install circup`` should do the trick. This is the simplest way to make it
@@ -39,7 +39,7 @@ If you have no idea what a virtualenv is, try the following command,
 
 .. note::
 
-    If you use the ``pip3`` command to install CircUp you must make sure that
+    If you use the ``pip3`` command to install Circup you must make sure that
     your path contains the directory into which the script will be installed.
     To discover this path,
 
@@ -76,7 +76,7 @@ Usage
 -----
 
 If you need more detailed help using Circup see the Learn Guide article
-`"Use CircUp to easily keep your CircuitPython libraries up to date" <https://learn.adafruit.com/keep-your-circuitpython-libraries-on-devices-up-to-date-with-circup/>`_.
+`"Use Circup to easily keep your CircuitPython libraries up to date" <https://learn.adafruit.com/keep-your-circuitpython-libraries-on-devices-up-to-date-with-circup/>`_.
 
 First, plug in a device running CircuiPython. This should appear as a mounted
 storage device called ``CIRCUITPY``.
@@ -230,7 +230,7 @@ The ``--version`` flag will tell you the current version of the
 ``circup`` command itself::
 
     $ circup --version
-    CircUp, A CircuitPython module updater. Version 0.0.1
+    Circup, A CircuitPython module updater. Version 0.0.1
 
 
 To use circup via the `Web Workflow <https://learn.adafruit.com/getting-started-with-web-workflow-using-the-code-editor>`_. on devices that support it. Use the ``--host`` and ``--password`` arguments before your circup command.::
@@ -263,6 +263,7 @@ For Bash, add this to ~/.bashrc::
 
 For Zsh, add this to ~/.zshrc::
 
+    autoload -U compinit; compinit
     eval "$(_CIRCUP_COMPLETE=zsh_source circup)"
 
 For Fish, add this to ~/.config/fish/completions/foo-bar.fish::

circup/__init__.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: MIT
 """
-CircUp -- a utility to manage and update libraries on a CircuitPython device.
+Circup -- a utility to manage and update libraries on a CircuitPython device.
 """
 
 

circup/backends.py

@@ -29,9 +29,10 @@ class Backend:
     implementations
     """
 
-    def __init__(self, logger):
+    def __init__(self, logger, version_override=None):
         self.device_location = None
         self.LIB_DIR_PATH = None
+        self.version_override = version_override
         self.logger = logger
 
     def get_circuitpython_version(self):
@@ -72,7 +73,7 @@ class Backend:
         """
         return self.get_modules(os.path.join(self.device_location, self.LIB_DIR_PATH))
 
-    def _create_library_directory(self, device_path, library_path):
+    def create_directory(self, device_path, directory):
         """
         To be overridden by subclass
         """
@@ -96,6 +97,12 @@ class Backend:
         """
         raise NotImplementedError
 
+    def upload_file(self, target_file, location_to_paste):
+        """Paste a copy of the specified file at the location given
+        To be overridden by subclass
+        """
+        raise NotImplementedError
+
     # pylint: disable=too-many-locals,too-many-branches,too-many-arguments,too-many-nested-blocks,too-many-statements
     def install_module(
         self, device_path, device_modules, name, pyext, mod_names, upgrade=False
@@ -188,7 +195,7 @@ class Backend:
                 return
 
             # Create the library directory first.
-            self._create_library_directory(device_path, library_path)
+            self.create_directory(device_path, library_path)
             if local_path is None:
                 if pyext:
                     # Use Python source for module.
@@ -226,6 +233,12 @@ class Backend:
         """
         raise NotImplementedError
 
+    def get_file_content(self, target_file):
+        """
+        To be overridden by subclass
+        """
+        raise NotImplementedError
+
     def get_free_space(self):
         """
         To be overridden by subclass
@@ -275,10 +288,14 @@ class WebBackend(Backend):
     Backend for interacting with a device via Web Workflow
     """
 
-    def __init__(self, host, password, logger, timeout=10):
+    def __init__(  # pylint: disable=too-many-arguments
+        self, host, port, password, logger, timeout=10, version_override=None
+    ):
         super().__init__(logger)
         if password is None:
-            raise ValueError("--host needs --password")
+            raise ValueError(
+                "Must pass --password or set CIRCUP_WEBWORKFLOW_PASSWORD environment variable"
+            )
 
         # pylint: disable=no-member
         # verify hostname/address
@@ -294,13 +311,19 @@ class WebBackend(Backend):
         self.FS_PATH = "fs/"
         self.LIB_DIR_PATH = f"{self.FS_PATH}lib/"
         self.host = host
+        self.port = port
         self.password = password
-        self.device_location = f"http://:{self.password}@{self.host}"
+        self.device_location = f"http://:{self.password}@{self.host}:{self.port}"
 
         self.session = requests.Session()
         self.session.mount(self.device_location, HTTPAdapter(max_retries=5))
         self.library_path = self.device_location + "/" + self.LIB_DIR_PATH
         self.timeout = timeout
+        self.version_override = version_override
+        self.FS_URL = urljoin(self.device_location, self.FS_PATH)
+
+    def __repr__(self):
+        return f"<WebBackend @{self.device_location}>"
 
     def install_file_http(self, source, location=None):
         """
@@ -401,6 +424,9 @@ class WebBackend(Backend):
 
         :return: A tuple with the version string for CircuitPython and the board ID string.
         """
+        if self.version_override is not None:
+            return self.version_override
+
         # pylint: disable=arguments-renamed
         with self.session.get(
             self.device_location + "/cp/version.json", timeout=self.timeout
@@ -535,10 +561,9 @@ class WebBackend(Backend):
             metadata["path"] = sfm_url
             result[sfm[:idx]] = metadata
 
-    def _create_library_directory(self, device_path, library_path):
-        url = urlparse(device_path)
-        auth = HTTPBasicAuth("", url.password)
-        with self.session.put(library_path, auth=auth, timeout=self.timeout) as r:
+    def create_directory(self, device_path, directory):
+        auth = HTTPBasicAuth("", self.password)
+        with self.session.put(directory, auth=auth, timeout=self.timeout) as r:
             if r.status_code == 409:
                 _writeable_error()
             r.raise_for_status()
@@ -549,11 +574,70 @@ class WebBackend(Backend):
                 self.device_location,
                 "/".join(("fs", location_to_paste, target_file, "")),
             )
-            self._create_library_directory(self.device_location, create_directory_url)
+            self.create_directory(self.device_location, create_directory_url)
             self.install_dir_http(target_file)
         else:
             self.install_file_http(target_file)
 
+    def upload_file(self, target_file, location_to_paste):
+        """
+        copy a file from the host PC to the microcontroller
+        :param target_file: file on the host PC to copy
+        :param location_to_paste: Location on the microcontroller to paste it.
+        :return:
+        """
+        if os.path.isdir(target_file):
+            create_directory_url = urljoin(
+                self.device_location,
+                "/".join(("fs", location_to_paste, target_file, "")),
+            )
+            self.create_directory(self.device_location, create_directory_url)
+            self.install_dir_http(target_file, location_to_paste)
+        else:
+            self.install_file_http(target_file, location_to_paste)
+
+    def download_file(self, target_file, location_to_paste):
+        """
+        Download a file from the MCU device to the local host PC
+        :param target_file: The file on the MCU to download
+        :param location_to_paste: The location on the host PC to put the downloaded copy.
+        :return:
+        """
+        auth = HTTPBasicAuth("", self.password)
+        with self.session.get(
+            self.FS_URL + target_file, timeout=self.timeout, auth=auth
+        ) as r:
+            if r.status_code == 404:
+                click.secho(f"{target_file} was not found on the device", "red")
+
+            file_name = target_file.split("/")[-1]
+            if location_to_paste is None:
+                with open(file_name, "wb") as f:
+                    f.write(r.content)
+
+                click.echo(f"Downloaded File: {file_name}")
+            else:
+                with open(os.path.join(location_to_paste, file_name), "wb") as f:
+                    f.write(r.content)
+
+                click.echo(
+                    f"Downloaded File: {os.path.join(location_to_paste, file_name)}"
+                )
+
+    def get_file_content(self, target_file):
+        """
+        Get the content of a file from the MCU drive
+        :param target_file: The file on the MCU to download
+        :return:
+        """
+        auth = HTTPBasicAuth("", self.password)
+        with self.session.get(
+            self.FS_URL + target_file, timeout=self.timeout, auth=auth
+        ) as r:
+            if r.status_code == 404:
+                return None
+            return r.content  # .decode("utf8")
+
     def install_module_mpy(self, bundle, metadata):
         """
         :param bundle library bundle.
@@ -591,19 +675,6 @@ class WebBackend(Backend):
         else:
             self.install_file_http(source_path, location=location)
 
-    def get_auto_file_path(self, auto_file_path):
-        """
-        Make a local temp copy of the --auto file from the device.
-        Returns the path to the local copy.
-        """
-        url = auto_file_path
-        auth = HTTPBasicAuth("", self.password)
-        with self.session.get(url, auth=auth, timeout=self.timeout) as r:
-            r.raise_for_status()
-            with open(LOCAL_CODE_PY_COPY, "w", encoding="utf-8") as f:
-                f.write(r.text)
-        return LOCAL_CODE_PY_COPY
-
     def uninstall(self, device_path, module_path):
         """
         Uninstall given module on device using REST API.
@@ -657,11 +728,7 @@ class WebBackend(Backend):
         """
         retuns the full path on the device to a given file name.
         """
-        return urljoin(
-            urljoin(self.device_location, "fs/", allow_fragments=False),
-            filename,
-            allow_fragments=False,
-        )
+        return "/".join((self.device_location, "fs", filename))
 
     def is_device_present(self):
         """
@@ -732,6 +799,19 @@ class WebBackend(Backend):
                 return r.json()["free"] * r.json()["block_size"]  # bytes
             sys.exit(1)
 
+    def list_dir(self, dirpath):
+        """
+        Returns the list of files located in the given dirpath.
+        """
+        auth = HTTPBasicAuth("", self.password)
+        with self.session.get(
+            urljoin(self.device_location, f"fs/{dirpath if dirpath else ''}"),
+            auth=auth,
+            headers={"Accept": "application/json"},
+            timeout=self.timeout,
+        ) as r:
+            return r.json()["files"]
+
 
 class DiskBackend(Backend):
     """
@@ -741,9 +821,10 @@ class DiskBackend(Backend):
     :param logger: logger to use for outputting messages
     :param String boot_out: Optional mock contents of a boot_out.txt file
         to use for version information.
+    :param String version_override: Optional mock version to use.
     """
 
-    def __init__(self, device_location, logger, boot_out=None):
+    def __init__(self, device_location, logger, boot_out=None, version_override=None):
         if device_location is None:
             raise ValueError(
                 "Auto locating USB Disk based device failed. "
@@ -757,6 +838,7 @@ class DiskBackend(Backend):
         self.version_info = None
         if boot_out is not None:
             self.version_info = self.parse_boot_out_file(boot_out)
+        self.version_override = version_override
 
     def get_circuitpython_version(self):
         """
@@ -773,6 +855,9 @@ class DiskBackend(Backend):
 
         :return: A tuple with the version string for CircuitPython and the board ID string.
         """
+        if self.version_override is not None:
+            return self.version_override
+
         if not self.version_info:
             try:
                 with open(
@@ -805,9 +890,9 @@ class DiskBackend(Backend):
         """
         return _get_modules_file(device_lib_path, self.logger)
 
-    def _create_library_directory(self, device_path, library_path):
-        if not os.path.exists(library_path):  # pragma: no cover
-            os.makedirs(library_path)
+    def create_directory(self, device_path, directory):
+        if not os.path.exists(directory):  # pragma: no cover
+            os.makedirs(directory)
 
     def copy_file(self, target_file, location_to_paste):
         target_filename = target_file.split(os.path.sep)[-1]
@@ -822,6 +907,9 @@ class DiskBackend(Backend):
                 os.path.join(self.device_location, location_to_paste, target_filename),
             )
 
+    def upload_file(self, target_file, location_to_paste):
+        self.copy_file(target_file, location_to_paste)
+
     def install_module_mpy(self, bundle, metadata):
         """
         :param bundle library bundle.
@@ -869,17 +957,14 @@ class DiskBackend(Backend):
             # Copy the directory.
             shutil.copytree(source_path, target_path)
         else:
-            target = os.path.basename(source_path)
+            if "target_name" in metadata:
+                target = metadata["target_name"]
+            else:
+                target = os.path.basename(source_path)
             target_path = os.path.join(location, target)
             # Copy file.
             shutil.copyfile(source_path, target_path)
 
-    def get_auto_file_path(self, auto_file_path):
-        """
-        Returns the path on the device to the file to be read for --auto.
-        """
-        return auto_file_path
-
     def uninstall(self, device_path, module_path):
         """
         Uninstall module using local file system.
@@ -930,6 +1015,18 @@ class DiskBackend(Backend):
         """
         return os.path.join(self.device_location, filename)
 
+    def get_file_content(self, target_file):
+        """
+        Get the content of a file from the MCU drive
+        :param target_file: The file on the MCU to download
+        :return:
+        """
+        file_path = self.get_file_path(target_file)
+        if os.path.exists(file_path):
+            with open(file_path, "rb") as file:
+                return file.read()
+        return None
+
     def is_device_present(self):
         """
         returns True if the device is currently connected
@@ -943,3 +1040,22 @@ class DiskBackend(Backend):
         # pylint: disable=unused-variable
         _, total, free = shutil.disk_usage(self.device_location)
         return free
+
+    def list_dir(self, dirpath):
+        """
+        Returns the list of files located in the given dirpath.
+        """
+        files_list = []
+        files = os.listdir(os.path.join(self.device_location, dirpath))
+        for file_name in files:
+            file = os.path.join(self.device_location, dirpath, file_name)
+            stat = os.stat(file)
+            files_list.append(
+                {
+                    "name": file_name,
+                    "directory": os.path.isdir(file),
+                    "modified_ns": stat.st_mtime_ns,
+                    "file_size": stat.st_size,
+                }
+            )
+        return files_list

circup/command_utils.py

@@ -5,6 +5,7 @@
 Functions called from commands in order to provide behaviors and return information.
 """
 
+import ast
 import ctypes
 import glob
 import os
@@ -16,7 +17,6 @@ import zipfile
 import json
 import re
 import toml
-import findimports
 import requests
 import click
 
@@ -41,6 +41,25 @@ WARNING_IGNORE_MODULES = (
     "circuitpython-typing",
 )
 
+CODE_FILES = [
+    "code.txt",
+    "code.py",
+    "main.py",
+    "main.txt",
+    "code.txt.py",
+    "code.py.txt",
+    "code.txt.txt",
+    "code.py.py",
+    "main.txt.py",
+    "main.py.txt",
+    "main.txt.txt",
+    "main.py.py",
+]
+
+
+class CodeParsingException(Exception):
+    """Exception thrown when parsing code with ast fails"""
+
 
 def clean_library_name(assumed_library_name):
     """
@@ -100,6 +119,7 @@ def completion_for_example(ctx, param, incomplete):
     Returns the list of available modules for the command line tab-completion
     with the ``circup example`` command.
     """
+
     # pylint: disable=unused-argument, consider-iterating-dictionary
     available_examples = get_bundle_examples(get_bundles_list(), avoid_download=True)
 
@@ -319,14 +339,22 @@ def get_bundle_examples(bundles_list, avoid_download=False):
     :return: A dictionary of metadata about the examples available in the
              library bundle.
     """
-    # pylint: disable=too-many-nested-blocks
+    # pylint: disable=too-many-nested-blocks,too-many-locals
     all_the_examples = dict()
+    bundle_examples = dict()
 
     try:
         for bundle in bundles_list:
             if not avoid_download or not os.path.isdir(bundle.lib_dir("py")):
                 ensure_latest_bundle(bundle)
             path = bundle.examples_dir("py")
+            meta_saved = os.path.join(path, "../bundle_examples.json")
+            if os.path.exists(meta_saved):
+                with open(meta_saved, "r", encoding="utf-8") as f:
+                    bundle_examples = json.load(f)
+                all_the_examples.update(bundle_examples)
+                bundle_examples.clear()
+                continue
             path_examples = _get_modules_file(path, logger)
             for lib_name, lib_metadata in path_examples.items():
                 for _dir_level in os.walk(lib_metadata["path"]):
@@ -337,8 +365,13 @@ def get_bundle_examples(bundles_list, avoid_download=False):
                         if _dirs[-1] == "":
                             _dirs.pop(-1)
                         slug = f"{os.path.sep}".join(_dirs + [_file.replace(".py", "")])
+                        bundle_examples[slug] = os.path.join(_dir_level[0], _file)
                         all_the_examples[slug] = os.path.join(_dir_level[0], _file)
 
+            with open(meta_saved, "w", encoding="utf-8") as f:
+                json.dump(bundle_examples, f)
+            bundle_examples.clear()
+
     except NotADirectoryError:
         # Bundle does not have new style examples directory
         # so we cannot include its examples.
@@ -591,26 +624,183 @@ def tags_data_save_tag(key, tag):
         json.dump(tags_data, data)
 
 
-def libraries_from_code_py(code_py, mod_names):
+def imports_from_code(full_content):
     """
     Parse the given code.py file and return the imported libraries
+    Note that it's impossible at that level to differentiate between
+    import module.property and import module.submodule, so we try both
 
-    :param str code_py: Full path of the code.py file
+    :param str full_content: Code to read imports from
+    :param str module_name: Name of the module the code is from
     :return: sequence of library names
     """
-    # pylint: disable=broad-except
+    # pylint: disable=too-many-branches
     try:
-        found_imports = findimports.find_imports(code_py)
-    except Exception as ex:  # broad exception because anything could go wrong
-        logger.exception(ex)
-        click.secho('Unable to read the auto file: "{}"'.format(str(ex)), fg="red")
+        par = ast.parse(full_content)
+    except (SyntaxError, ValueError) as err:
+        raise CodeParsingException(err) from err
+
+    imports = set()
+    for thing in ast.walk(par):
+        # import module and import module.submodule
+        if isinstance(thing, ast.Import):
+            for alias in thing.names:
+                imports.add(alias.name)
+        # from x import y
+        if isinstance(thing, ast.ImportFrom):
+            if thing.module:
+                # from [.][.]module import names
+                module = ("." * thing.level) + thing.module
+                imports.add(module)
+                for alias in thing.names:
+                    imports.add(".".join([module, alias.name]))
+            else:
+                # from . import names
+                for alias in thing.names:
+                    imports.add(alias.name)
+
+    # import parent modules (in practice it's the __init__.py)
+    for name in list(imports):
+        if "*" in name:
+            imports.remove(name)
+            continue
+        names = name.split(".")
+        for i in range(len(names)):
+            module = ".".join(names[: i + 1])
+            if module:
+                imports.add(module)
+
+    return sorted(imports)
+
+
+def get_all_imports(  # pylint: disable=too-many-arguments,too-many-locals, too-many-branches
+    backend, auto_file_content, auto_file_path, mod_names, current_module, visited=None
+):
+    """
+    Recursively retrieve imports from files on the backend
+
+    :param Backend backend: The current backend object
+    :param str auto_file_content: Content of the python file to analyse
+    :param str auto_file_path: Path to the python file to analyse
+    :param list mod_names: Lits of supported bundle mod names
+    :param str current_module: Name of the call context module if recursive call
+    :param set visited: Modules previously visited
+    :return: sequence of library names
+    """
+    if visited is None:
+        visited = set()
+    visited.add(current_module)
+
+    requested_installs = []
+    try:
+        imports = imports_from_code(auto_file_content)
+    except CodeParsingException as err:
+        click.secho(f"Error parsing {current_module}:\n  {err}", fg="red")
         sys.exit(2)
-    # pylint: enable=broad-except
-    imports = [info.name.split(".", 1)[0] for info in found_imports]
-    return [r for r in imports if r in mod_names]
+
+    for install in imports:
+        if install in visited:
+            continue
+        if install in mod_names:
+            requested_installs.append(install)
+        else:
+            # relative module paths
+            if install.startswith(".."):
+                install_module = ".".join(current_module.split(".")[:-2])
+                install_module = install_module + "." + install[2:]
+            elif install.startswith("."):
+                install_module = ".".join(current_module.split(".")[:-1])
+                install_module = install_module + "." + install[1:]
+            else:
+                install_module = install
+            # possible files for the module: .py or __init__.py (if directory)
+            file_name = os.path.join(*install_module.split(".")) + ".py"
+            try:
+                file_location = os.path.join(
+                    *auto_file_path.replace(str(backend.device_location), "").split(
+                        "/"
+                    )[:-1]
+                )
+
+                full_location = os.path.join(file_location, file_name)
+
+            except TypeError:
+                # file is in root of CIRCUITPY
+                full_location = file_name
+
+            exists = backend.file_exists(full_location)
+            if not exists:
+                file_name = os.path.join(*install_module.split("."), "__init__.py")
+                full_location = file_name
+                exists = backend.file_exists(full_location)
+                if not exists:
+                    continue
+                install_module += ".__init__"
+            # get the content and parse it recursively
+            auto_file_content = backend.get_file_content(full_location)
+            if auto_file_content:
+                sub_imports = get_all_imports(
+                    backend,
+                    auto_file_content,
+                    auto_file_path,
+                    mod_names,
+                    install_module,
+                    visited,
+                )
+                requested_installs.extend(sub_imports)
+
+    return sorted(requested_installs)
+    # [r for r in requested_installs if r in mod_names]
 
 
-def get_device_path(host, password, path):
+def libraries_from_auto_file(backend, auto_file, mod_names):
+    """
+    Parse the input auto_file path and/or use the workflow to find the most
+    appropriate code.py script. Then return the list of imports
+
+    :param Backend backend: The current backend object
+    :param str auto_file: Path of the candidate auto file or None
+    :return: sequence of library names
+    """
+    # find the current main file based on Circuitpython's rules
+    if auto_file is None:
+        root_files = [
+            file["name"] for file in backend.list_dir("") if not file["directory"]
+        ]
+        for main_file in CODE_FILES:
+            if main_file in root_files:
+                auto_file = main_file
+                break
+    # still no code file found
+    if auto_file is None:
+        click.secho(
+            "No default code file found. See valid names:\n"
+            "https://docs.circuitpython.org/en/latest/README.html#behavior",
+            fg="red",
+        )
+        sys.exit(1)
+
+    # pass a local file with "./" or "../"
+    is_relative = auto_file.split(os.sep)[0] in [os.path.curdir, os.path.pardir]
+    if os.path.isabs(auto_file) or is_relative:
+        with open(auto_file, "r", encoding="UTF8") as fp:
+            auto_file_content = fp.read()
+    else:
+        auto_file_content = backend.get_file_content(auto_file)
+
+    if auto_file_content is None:
+        click.secho(f"Auto file not found: {auto_file}", fg="red")
+        sys.exit(1)
+
+    # from file name to module name (in case it's in a subpackage)
+    click.secho(f"Finding imports from: {auto_file}", fg="green")
+    current_module = auto_file.rstrip(".py").replace(os.path.sep, ".")
+    return get_all_imports(
+        backend, auto_file_content, auto_file, mod_names, current_module
+    )
+
+
+def get_device_path(host, port, password, path):
     """
     :param host Hostname or IP address.
     :param password REST API password.
@@ -621,7 +811,33 @@ def get_device_path(host, password, path):
         device_path = path
     elif host:
         # pylint: enable=no-member
-        device_path = f"http://:{password}@" + host
+        device_path = f"http://:{password}@{host}:{port}"
     else:
         device_path = find_device()
     return device_path
+
+
+def sorted_by_directory_then_alpha(list_of_files):
+    """
+    Sort the list of files into alphabetical seperated
+    with directories grouped together before files.
+    """
+    dirs = {}
+    files = {}
+
+    for cur_file in list_of_files:
+        if cur_file["directory"]:
+            dirs[cur_file["name"]] = cur_file
+        else:
+            files[cur_file["name"]] = cur_file
+
+    sorted_dir_names = sorted(dirs.keys())
+    sorted_file_names = sorted(files.keys())
+
+    sorted_full_list = []
+    for cur_name in sorted_dir_names:
+        sorted_full_list.append(dirs[cur_name])
+    for cur_name in sorted_file_names:
+        sorted_full_list.append(files[cur_name])
+
+    return sorted_full_list

circup/commands.py

@@ -11,6 +11,7 @@ and the respective Backends which *are* tested. Most of the logic of the followi
 functions is to prepare things for presentation to / interaction with the user.
 """
 import os
+import subprocess
 import time
 import sys
 import re
@@ -33,7 +34,7 @@ from circup.command_utils import (
     completion_for_install,
     get_bundle_versions,
     libraries_from_requirements,
-    libraries_from_code_py,
+    libraries_from_auto_file,
     get_dependencies,
     get_bundles_local_dict,
     save_local_bundles,
@@ -57,7 +58,13 @@ from circup.command_utils import (
     help="Hostname or IP address of a device. Overrides automatic path detection.",
 )
 @click.option(
-    "--password", help="Password to use for authentication when --host is used."
+    "--port", help="Port to contact. Overrides automatic path detection.", default=80
+)
+@click.option(
+    "--password",
+    help="Password to use for authentication when --host is used."
+    " You can optionally set an environment variable CIRCUP_WEBWORKFLOW_PASSWORD"
+    " instead of passing this argument. If both exist the CLI arg takes precedent.",
 )
 @click.option(
     "--timeout",
@@ -77,20 +84,24 @@ from circup.command_utils import (
     "with --board-id, it overrides the detected CPy version.",
 )
 @click.version_option(
-    prog_name="CircUp",
+    prog_name="Circup",
     message="%(prog)s, A CircuitPython module updater. Version %(version)s",
 )
 @click.pass_context
 def main(  # pylint: disable=too-many-locals
-    ctx, verbose, path, host, password, timeout, board_id, cpy_version
+    ctx, verbose, path, host, port, password, timeout, board_id, cpy_version
 ):  # pragma: no cover
     """
     A tool to manage and update libraries on a CircuitPython device.
     """
-    # pylint: disable=too-many-arguments,too-many-branches,too-many-statements,too-many-locals
+    # pylint: disable=too-many-arguments,too-many-branches,too-many-statements,too-many-locals, R0801
     ctx.ensure_object(dict)
     ctx.obj["TIMEOUT"] = timeout
-    device_path = get_device_path(host, password, path)
+
+    if password is None:
+        password = os.getenv("CIRCUP_WEBWORKFLOW_PASSWORD")
+
+    device_path = get_device_path(host, port, password, path)
 
     using_webworkflow = "host" in ctx.params.keys() and ctx.params["host"] is not None
 
@@ -105,7 +116,12 @@ def main(  # pylint: disable=too-many-locals
             device_path = device_path.replace("circuitpython.local", host)
         try:
             ctx.obj["backend"] = WebBackend(
-                host=host, password=password, logger=logger, timeout=timeout
+                host=host,
+                port=port,
+                password=password,
+                logger=logger,
+                timeout=timeout,
+                version_override=cpy_version,
             )
         except ValueError as e:
             click.secho(e, fg="red")
@@ -116,7 +132,11 @@ def main(  # pylint: disable=too-many-locals
             sys.exit(1)
     else:
         try:
-            ctx.obj["backend"] = DiskBackend(device_path, logger)
+            ctx.obj["backend"] = DiskBackend(
+                device_path,
+                logger,
+                version_override=cpy_version,
+            )
         except ValueError as e:
             print(e)
 
@@ -158,8 +178,8 @@ def main(  # pylint: disable=too-many-locals
             else (cpy_version, board_id)
         )
         click.echo(
-            "Found device at {}, running CircuitPython {}.".format(
-                device_path, cpy_version
+            "Found device {} at {}, running CircuitPython {}.".format(
+                board_id, device_path, cpy_version
             )
         )
     try:
@@ -289,6 +309,12 @@ def list_cli(ctx):  # pragma: no cover
 @click.option(
     "--upgrade", "-U", is_flag=True, help="Upgrade modules that are already installed."
 )
+@click.option(
+    "--stubs",
+    "-s",
+    is_flag=True,
+    help="Install stubs module from PyPi for context in IDE.",
+)
 @click.option(
     "--auto-file",
     default=None,
@@ -297,7 +323,7 @@ def list_cli(ctx):  # pragma: no cover
 )
 @click.pass_context
 def install(
-    ctx, modules, pyext, requirement, auto, auto_file, upgrade=False
+    ctx, modules, pyext, requirement, auto, auto_file, upgrade=False, stubs=False
 ):  # pragma: no cover
     """
     Install a named module(s) onto the device. Multiple modules
@@ -305,6 +331,7 @@ def install(
     separated by a space. Modules can be from a Bundle or local filepaths.
     """
 
+    # pylint: disable=too-many-branches
     # TODO: Ensure there's enough space on the device
     available_modules = get_bundle_versions(get_bundles_list())
     mod_names = {}
@@ -315,32 +342,12 @@ def install(
             requirements_txt = rfile.read()
         requested_installs = libraries_from_requirements(requirements_txt)
     elif auto or auto_file:
-        if auto_file is None:
-            auto_file = "code.py"
-            print(f"Auto file: {auto_file}")
-        # pass a local file with "./" or "../"
-        is_relative = not isinstance(ctx.obj["backend"], WebBackend) or auto_file.split(
-            os.sep
-        )[0] in [os.path.curdir, os.path.pardir]
-        if not os.path.isabs(auto_file) and not is_relative:
-            auto_file = ctx.obj["backend"].get_file_path(auto_file or "code.py")
-
-        auto_file_path = ctx.obj["backend"].get_auto_file_path(auto_file)
-        print(f"Auto file path: {auto_file_path}")
-        if not os.path.isfile(auto_file_path):
-            # fell through to here when run from random folder on windows - ask backend.
-            new_auto_file = ctx.obj["backend"].get_file_path(auto_file)
-            if os.path.isfile(new_auto_file):
-                auto_file = new_auto_file
-                auto_file_path = ctx.obj["backend"].get_auto_file_path(auto_file)
-                print(f"Auto file path: {auto_file_path}")
-            else:
-                click.secho(f"Auto file not found: {auto_file}", fg="red")
-                sys.exit(1)
-
-        requested_installs = libraries_from_code_py(auto_file_path, mod_names)
+        requested_installs = libraries_from_auto_file(
+            ctx.obj["backend"], auto_file, mod_names
+        )
     else:
         requested_installs = modules
+
     requested_installs = sorted(set(requested_installs))
     click.echo(f"Searching for dependencies for: {requested_installs}")
     to_install = get_dependencies(requested_installs, mod_names=mod_names)
@@ -358,34 +365,74 @@ def install(
                 upgrade,
             )
 
+            if stubs:
+                library_stubs = "adafruit-circuitpython-{}".format(
+                    library.replace("adafruit_", "")
+                )
+                try:
+                    output = subprocess.check_output(["pip", "install", library_stubs])
+                    if (
+                        f"Requirement already satisfied: {library_stubs}"
+                        in output.decode()
+                    ):
+                        click.echo(f"'{library}' stubs already installed.")
+                    else:
+                        click.echo(f"Installed '{library}' stubs.")
+                except subprocess.CalledProcessError:
+                    click.secho(
+                        f"Could not install stubs module {library_stubs}", fg="yellow"
+                    )
+
 
 @main.command()
 @click.option("--overwrite", is_flag=True, help="Overwrite the file if it exists.")
+@click.option("--list", "-ls", "op_list", is_flag=True, help="List available examples.")
+@click.option("--rename", is_flag=True, help="Install the example as code.py.")
 @click.argument(
-    "examples", required=True, nargs=-1, shell_complete=completion_for_example
+    "examples", required=False, nargs=-1, shell_complete=completion_for_example
 )
 @click.pass_context
-def example(ctx, examples, overwrite):
+def example(ctx, examples, op_list, rename, overwrite):
     """
     Copy named example(s) from a bundle onto the device. Multiple examples
     can be installed at once by providing more than one example name, each
     separated by a space.
     """
 
+    if op_list:
+        if examples:
+            click.echo("\n".join(completion_for_example(ctx, "", examples)))
+        else:
+            click.echo("Available example libraries:")
+            available_examples = get_bundle_examples(
+                get_bundles_list(), avoid_download=True
+            )
+            lib_names = {
+                str(key.split(os.path.sep)[0]): value
+                for key, value in available_examples.items()
+            }
+            click.echo("\n".join(sorted(lib_names.keys())))
+        return
+
     for example_arg in examples:
         available_examples = get_bundle_examples(
             get_bundles_list(), avoid_download=True
         )
         if example_arg in available_examples:
             filename = available_examples[example_arg].split(os.path.sep)[-1]
+            install_metadata = {"path": available_examples[example_arg]}
+
+            filename = available_examples[example_arg].split(os.path.sep)[-1]
+            if rename:
+                if os.path.isfile(available_examples[example_arg]):
+                    filename = "code.py"
+                    install_metadata["target_name"] = filename
 
             if overwrite or not ctx.obj["backend"].file_exists(filename):
                 click.echo(
                     f"{'Copying' if not overwrite else 'Overwriting'}: {filename}"
                 )
-                ctx.obj["backend"].install_module_py(
-                    {"path": available_examples[example_arg]}, location=""
-                )
+                ctx.obj["backend"].install_module_py(install_metadata, location="")
             else:
                 click.secho(
                     f"File: {filename} already exists. Use --overwrite if you wish to replace it.",
@@ -682,7 +729,7 @@ def bundle_remove(bundle, reset):
     bundles_local_dict = get_bundles_local_dict()
     modified = False
     for bun in bundle:
-        # cleanup in case seombody pastes the URL to the repo/releases
+        # cleanup in case somebody pastes the URL to the repo/releases
         bun = re.sub(r"https?://github.com/([^/]+/[^/]+)(/.*)?", r"\1", bun)
         found = False
         for name, repo in list(bundles_local_dict.items()):

circup/config/bundle_config.json

@@ -1,5 +1,4 @@
 {
     "adafruit": "adafruit/Adafruit_CircuitPython_Bundle",
-    "circuitpython_community": "adafruit/CircuitPython_Community_Bundle",
-    "circuitpython_org": "circuitpython/CircuitPython_Org_Bundle"
+    "circuitpython_community": "adafruit/CircuitPython_Community_Bundle"
 }

circup/shared.py

@@ -10,8 +10,8 @@ import glob
 import os
 import re
 import json
+import importlib.resources
 import appdirs
-import pkg_resources
 import requests
 
 #: Version identifier for a bad MPY file format
@@ -21,15 +21,14 @@ BAD_FILE_FORMAT = "Invalid"
 DATA_DIR = appdirs.user_data_dir(appname="circup", appauthor="adafruit")
 
 #: Module formats list (and the other form used in github files)
-PLATFORMS = {"py": "py", "8mpy": "8.x-mpy", "9mpy": "9.x-mpy"}
+PLATFORMS = {"py": "py", "9mpy": "9.x-mpy", "10mpy": "10.x-mpy"}
 
 #: Timeout for requests calls like get()
 REQUESTS_TIMEOUT = 30
 
 #: The path to the JSON file containing the metadata about the bundles.
-BUNDLE_CONFIG_FILE = pkg_resources.resource_filename(
-    "circup", "config/bundle_config.json"
-)
+BUNDLE_CONFIG_FILE = importlib.resources.files("circup") / "config/bundle_config.json"
+
 #: Overwrite the bundles list with this file (only done manually)
 BUNDLE_CONFIG_OVERWRITE = os.path.join(DATA_DIR, "bundle_config.json")
 #: The path to the JSON file containing the local list of bundles.
@@ -80,14 +79,18 @@ def _get_modules_file(path, logger):
         py_files = glob.glob(os.path.join(package_path, "**/*.py"), recursive=True)
         mpy_files = glob.glob(os.path.join(package_path, "**/*.mpy"), recursive=True)
         all_files = py_files + mpy_files
+        # put __init__ first if any, assumed to have the version number
+        all_files.sort()
         # default value
         result[name] = {"path": package_path, "mpy": bool(mpy_files)}
         # explore all the submodules to detect bad ones
         for source in [f for f in all_files if not os.path.basename(f).startswith(".")]:
             metadata = extract_metadata(source, logger)
             if "__version__" in metadata:
-                metadata["path"] = package_path
-                result[name] = metadata
+                # don't replace metadata if already found
+                if "__version__" not in result[name]:
+                    metadata["path"] = package_path
+                    result[name] = metadata
                 # break now if any of the submodules has a bad format
                 if metadata["__version__"] == BAD_FILE_FORMAT:
                     break

circup/wwshell/README.rst

@@ -0,0 +1,105 @@
+
+wwshell
+=======
+
+.. image:: https://readthedocs.org/projects/circup/badge/?version=latest
+    :target: https://circuitpython.readthedocs.io/projects/circup/en/latest/
+    :alt: Documentation Status
+
+.. image:: https://img.shields.io/discord/327254708534116352.svg
+    :target: https://adafru.it/discord
+    :alt: Discord
+
+
+.. image:: https://github.com/adafruit/circup/workflows/Build%20CI/badge.svg
+    :target: https://github.com/adafruit/circup/actions
+    :alt: Build Status
+
+
+.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
+    :target: https://github.com/psf/black
+    :alt: Code Style: Black
+
+
+A tool to manage files on a CircuitPython device via wireless workflows.
+Currently supports Web Workflow.
+
+.. contents::
+
+Installation
+------------
+
+wwshell is bundled along with Circup. When you install Circup you'll get wwshell automatically.
+
+Circup requires Python 3.5 or higher.
+
+In a `virtualenv <https://virtualenv.pypa.io/en/latest/>`_,
+``pip install circup`` should do the trick. This is the simplest way to make it
+work.
+
+If you have no idea what a virtualenv is, try the following command,
+``pip3 install --user circup``.
+
+.. note::
+
+    If you use the ``pip3`` command to install CircUp you must make sure that
+    your path contains the directory into which the script will be installed.
+    To discover this path,
+
+    * On Unix-like systems, type ``python3 -m site --user-base`` and append
+      ``bin`` to the resulting path.
+    * On Windows, type the same command, but append ``Scripts`` to the
+      resulting path.
+
+What does wwshell do?
+---------------------
+
+It lets you view, delete, upload, and download files from your Circuitpython device
+via wireless workflows. Similar to ampy, but operates over wireless workflow rather
+than USB serial.
+
+Usage
+-----
+
+To use web workflow you need to enable it by putting WIFI credentials and a web workflow
+password into your settings.toml file. `See here <https://learn.adafruit.com/getting-started-with-web-workflow-using-the-code-editor/device-setup>`_,
+
+To get help, just type the command::
+
+    $ wwshell
+    Usage: wwshell [OPTIONS] COMMAND [ARGS]...
+
+      A tool to manage files CircuitPython device over web workflow.
+
+    Options:
+      --verbose          Comprehensive logging is sent to stdout.
+      --path DIRECTORY   Path to CircuitPython directory. Overrides automatic path
+                         detection.
+      --host TEXT        Hostname or IP address of a device. Overrides automatic
+                         path detection.
+      --password TEXT    Password to use for authentication when --host is used.
+                         You can optionally set an environment variable
+                         CIRCUP_WEBWORKFLOW_PASSWORD instead of passing this
+                         argument. If both exist the CLI arg takes precedent.
+      --timeout INTEGER  Specify the timeout in seconds for any network
+                         operations.
+      --version          Show the version and exit.
+      --help             Show this message and exit.
+
+    Commands:
+      get  Download a copy of a file or directory from the device to the...
+      ls   Lists the contents of a directory.
+      put  Upload a copy of a file or directory from the local computer to...
+      rm   Delete a file on the device.
+
+
+.. note::
+
+    If you find a bug, or you want to suggest an enhancement or new feature
+    feel free to create an issue or submit a pull request here:
+
+    https://github.com/adafruit/circup
+
+
+Discussion of this tool happens on the Adafruit CircuitPython
+`Discord channel <https://discord.gg/rqrKDjU>`_.

circup/wwshell/README.rst.license

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2024 Tim Cocks, written for Adafruit Industries
+#
+# SPDX-License-Identifier: MIT

circup/wwshell/__init__.py

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: 2024 Tim Cocks, written for Adafruit Industries
+#
+# SPDX-License-Identifier: MIT
+"""
+wwshell is a CLI utility for managing files on CircuitPython devices via wireless workflows.
+It currently supports Web Workflow.
+"""
+from .commands import main
+
+
+# Allows execution via `python -m circup ...`
+# pylint: disable=no-value-for-parameter
+if __name__ == "__main__":
+    main()

circup/wwshell/commands.py

@@ -0,0 +1,231 @@
+# SPDX-FileCopyrightText: 2019 Nicholas Tollervey, 2024 Tim Cocks, written for Adafruit Industries
+#
+# SPDX-License-Identifier: MIT
+"""
+# ----------- CLI command definitions  ----------- #
+
+The following functions have IO side effects (for instance they emit to
+stdout). Ergo, these are not checked with unit tests. Most of the
+functionality they provide is provided by the functions from util_functions.py,
+and the respective Backends which *are* tested. Most of the logic of the following
+functions is to prepare things for presentation to / interaction with the user.
+"""
+import os
+import time
+import sys
+import logging
+import update_checker
+import click
+import requests
+
+
+from circup.backends import WebBackend
+from circup.logging import logger, log_formatter, LOGFILE
+from circup.shared import BOARDLESS_COMMANDS
+
+from circup.command_utils import (
+    get_device_path,
+    get_circup_version,
+    sorted_by_directory_then_alpha,
+)
+
+
+@click.group()
+@click.option(
+    "--verbose", is_flag=True, help="Comprehensive logging is sent to stdout."
+)
+@click.option(
+    "--path",
+    type=click.Path(exists=True, file_okay=False),
+    help="Path to CircuitPython directory. Overrides automatic path detection.",
+)
+@click.option(
+    "--host",
+    help="Hostname or IP address of a device. Overrides automatic path detection.",
+    default="circuitpython.local",
+)
+@click.option(
+    "--port",
+    help="HTTP port that the web workflow is listening on.",
+    default=80,
+)
+@click.option(
+    "--password",
+    help="Password to use for authentication when --host is used."
+    " You can optionally set an environment variable CIRCUP_WEBWORKFLOW_PASSWORD"
+    " instead of passing this argument. If both exist the CLI arg takes precedent.",
+)
+@click.option(
+    "--timeout",
+    default=30,
+    help="Specify the timeout in seconds for any network operations.",
+)
+@click.version_option(
+    prog_name="CircFile",
+    message="%(prog)s, A CircuitPython web workflow file managemenr. Version %(version)s",
+)
+@click.pass_context
+def main(  # pylint: disable=too-many-locals
+    ctx,
+    verbose,
+    path,
+    host,
+    port,
+    password,
+    timeout,
+):  # pragma: no cover
+    """
+    A tool to manage files CircuitPython device over web workflow.
+    """
+    # pylint: disable=too-many-arguments,too-many-branches,too-many-statements,too-many-locals, R0801
+    ctx.ensure_object(dict)
+    ctx.obj["TIMEOUT"] = timeout
+
+    if password is None:
+        password = os.getenv("CIRCUP_WEBWORKFLOW_PASSWORD")
+
+    device_path = get_device_path(host, port, password, path)
+
+    using_webworkflow = "host" in ctx.params.keys() and ctx.params["host"] is not None
+    if using_webworkflow:
+        if host == "circuitpython.local":
+            click.echo("Checking versions.json on circuitpython.local to find hostname")
+            versions_resp = requests.get(
+                "http://circuitpython.local/cp/version.json", timeout=timeout
+            )
+            host = f'{versions_resp.json()["hostname"]}.local'
+            click.echo(f"Using hostname: {host}")
+            device_path = device_path.replace("circuitpython.local", host)
+        try:
+            ctx.obj["backend"] = WebBackend(
+                host=host, port=port, password=password, logger=logger, timeout=timeout
+            )
+        except ValueError as e:
+            click.secho(e, fg="red")
+            time.sleep(0.3)
+            sys.exit(1)
+        except RuntimeError as e:
+            click.secho(e, fg="red")
+            sys.exit(1)
+
+    if verbose:
+        # Configure additional logging to stdout.
+        ctx.obj["verbose"] = True
+        verbose_handler = logging.StreamHandler(sys.stdout)
+        verbose_handler.setLevel(logging.INFO)
+        verbose_handler.setFormatter(log_formatter)
+        logger.addHandler(verbose_handler)
+        click.echo("Logging to {}\n".format(LOGFILE))
+    else:
+        ctx.obj["verbose"] = False
+
+    logger.info("### Started Circfile ###")
+
+    # If a newer version of circfile is available, print a message.
+    logger.info("Checking for a newer version of circfile")
+    version = get_circup_version()
+    if version:
+        update_checker.update_check("circfile", version)
+
+    # stop early if the command is boardless
+    if ctx.invoked_subcommand in BOARDLESS_COMMANDS or "--help" in sys.argv:
+        return
+
+    ctx.obj["DEVICE_PATH"] = device_path
+
+    if device_path is None or not ctx.obj["backend"].is_device_present():
+        click.secho("Could not find a connected CircuitPython device.", fg="red")
+        sys.exit(1)
+    else:
+        click.echo("Found device at {}.".format(device_path))
+
+
+@main.command("ls")
+@click.argument("file", required=True, nargs=1, default="/")
+@click.pass_context
+def ls_cli(ctx, file):  # pragma: no cover
+    """
+    Lists the contents of a directory. Defaults to root directory
+    if not supplied.
+    """
+    logger.info("ls")
+    if not file.endswith("/"):
+        file += "/"
+    click.echo(f"running: ls {file}")
+
+    files = ctx.obj["backend"].list_dir(file)
+    click.echo("Size\tName")
+    for cur_file in sorted_by_directory_then_alpha(files):
+        click.echo(
+            f"{cur_file['file_size']}\t{cur_file['name']}{'/' if cur_file['directory'] else ''}"
+        )
+
+
+@main.command("put")
+@click.argument("file", required=True, nargs=1)
+@click.argument("location", required=False, nargs=1, default="")
+@click.option("--overwrite", is_flag=True, help="Overwrite the file if it exists.")
+@click.pass_context
+def put_cli(ctx, file, location, overwrite):
+    """
+    Upload a copy of a file or directory from the local computer
+    to the device
+    """
+    click.echo(f"Attempting PUT: {file} at {location} overwrite? {overwrite}")
+    if not ctx.obj["backend"].file_exists(f"{location}{file}"):
+        ctx.obj["backend"].upload_file(file, location)
+        click.echo(f"Successfully PUT {location}{file}")
+    else:
+        if overwrite:
+            click.secho(
+                f"{location}{file} already exists. Overwriting it.", fg="yellow"
+            )
+            ctx.obj["backend"].upload_file(file, location)
+            click.echo(f"Successfully PUT {location}{file}")
+        else:
+            click.secho(
+                f"{location}{file} already exists. Pass --overwrite if you wish to replace it.",
+                fg="red",
+            )
+
+
+# pylint: enable=too-many-arguments,too-many-locals
+
+
+@main.command("get")
+@click.argument("file", required=True, nargs=1)
+@click.argument("location", required=False, nargs=1)
+@click.pass_context
+def get_cli(ctx, file, location):  # pragma: no cover
+    """
+    Download a copy of a file or directory from the device to the local computer.
+    """
+
+    click.echo(f"running: get {file} {location}")
+    ctx.obj["backend"].download_file(file, location)
+
+
+@main.command("rm")
+@click.argument("file", nargs=1)
+@click.pass_context
+def rm_cli(ctx, file):  # pragma: no cover
+    """
+    Delete a file on the device.
+    """
+    click.echo(f"running: rm {file}")
+    ctx.obj["backend"].uninstall(
+        ctx.obj["backend"].device_location, ctx.obj["backend"].get_file_path(file)
+    )
+
+
+@main.command("mkdir")
+@click.argument("directory", nargs=1)
+@click.pass_context
+def mkdir_cli(ctx, directory):  # pragma: no cover
+    """
+    Create
+    """
+    click.echo(f"running: mkdir {directory}")
+    ctx.obj["backend"].create_directory(
+        ctx.obj["backend"].device_location, ctx.obj["backend"].get_file_path(directory)
+    )

docs/conf.py

@@ -109,7 +109,6 @@ if not on_rtd:  # only import and set the theme if we're building docs locally
         import sphinx_rtd_theme
 
         html_theme = "sphinx_rtd_theme"
-        html_theme_path = [sphinx_rtd_theme.get_html_theme_path(), "."]
     except:
         html_theme = "default"
         html_theme_path = ["."]

docs/index.rst

@@ -1,10 +1,12 @@
-.. CircUp documentation master file, created by
+.. Circup documentation master file, created by
    sphinx-quickstart on Mon Sep  2 10:58:36 2019.
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
 .. include:: ../README.rst
 
+.. include:: ../circup/wwshell/README.rst
+
 .. include:: ../CONTRIBUTING.rst
 
 API

pyproject.toml

@@ -0,0 +1,52 @@
+# SPDX-FileCopyrightText: 2024 Jev Kuznetsov, ROX Automation
+#
+# SPDX-License-Identifier: MIT
+
+[build-system]
+requires = ["setuptools>=61.0", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "circup"
+dynamic = ["version", "dependencies", "optional-dependencies"]
+description = "A tool to manage/update libraries on CircuitPython devices."
+readme = "README.rst"
+authors = [{ name = "Adafruit Industries", email = "circuitpython@adafruit.com" }]
+license = { file = "LICENSE" }
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Education",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: POSIX",
+    "Operating System :: MacOS :: MacOS X",
+    "Operating System :: Microsoft :: Windows",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Topic :: Education",
+    "Topic :: Software Development :: Embedded Systems",
+    "Topic :: System :: Software Distribution"
+]
+keywords = ["adafruit", "blinka", "circuitpython", "micropython", "libraries"]
+
+requires-python = ">=3.9"
+
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
+optional-dependencies = {optional = {file = ["optional_requirements.txt"]}}
+
+[tool.setuptools_scm]
+
+[project.scripts]
+circup = "circup:main"
+wwshell = "circup.wwshell:main"
+
+[project.urls]
+homepage = "https://github.com/adafruit/circup"
+
+[tool.setuptools.packages.find]
+where = ["."]  # This tells setuptools to look in the project root directory
+include = ["circup"]  # This pattern includes your main package and any sub-packages within it

requirements.txt

@@ -1,6 +1,5 @@
 appdirs
 Click
-findimports
 requests
 semver
 toml

setup.py

@@ -1,104 +0,0 @@
-# SPDX-FileCopyrightText: 2019 Nicholas Tollervey, written for Adafruit Industries
-#
-# SPDX-License-Identifier: MIT
-
-
-"""A setuptools based setup module.
-See:
-https://packaging.python.org/guides/distributing-packages-using-setuptools/
-https://github.com/pypa/sampleproject
-"""
-
-# Always prefer setuptools over distutils
-from setuptools import setup, find_packages
-
-# To use a consistent encoding
-from codecs import open
-from os import path
-
-here = path.abspath(path.dirname(__file__))
-
-# Get the long description from the README file
-with open(path.join(here, "README.rst"), encoding="utf-8") as f:
-    long_description = f.read()
-
-install_requires = [
-    "semver~=3.0",
-    "Click>=8.0",
-    "appdirs>=1.4.3",
-    "requests>=2.22.0",
-    "findimports>=2.1.0",
-    "toml>=0.10.2",
-    # importlib_metadata is only available for 3.7, and is not needed for 3.8 and up.
-    "importlib_metadata; python_version == '3.7'",
-    "update_checker",
-]
-
-extras_require = {
-    "tests": [
-        "pytest",
-        "pylint",
-        "pytest-cov",
-        "pytest-random-order>=1.0.0",
-        "pytest-faulthandler",
-        "coverage",
-        "black",
-    ],
-    "docs": ["sphinx"],
-    "package": [
-        # Wheel building and PyPI uploading
-        "wheel",
-        "twine",
-    ],
-}
-
-extras_require["dev"] = (
-    extras_require["tests"] + extras_require["docs"] + extras_require["package"]
-)
-
-extras_require["all"] = list(
-    {req for extra, reqs in extras_require.items() for req in reqs}
-)
-
-setup(
-    name="circup",
-    use_scm_version=True,
-    setup_requires=["setuptools_scm"],
-    description="A tool to manage/update libraries on CircuitPython devices.",
-    long_description=long_description,
-    long_description_content_type="text/x-rst",
-    # The project's main homepage.
-    url="https://github.com/adafruit/circup",
-    # Author details
-    author="Adafruit Industries",
-    author_email="circuitpython@adafruit.com",
-    install_requires=install_requires,
-    extras_require=extras_require,
-    # Choose your license
-    license="MIT",
-    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
-    classifiers=[
-        "Development Status :: 3 - Alpha",
-        "Environment :: Console",
-        "Intended Audience :: Developers",
-        "Intended Audience :: Education",
-        "License :: OSI Approved :: MIT License",
-        "Operating System :: POSIX",
-        "Operating System :: MacOS :: MacOS X",
-        "Operating System :: Microsoft :: Windows",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Topic :: Education",
-        "Topic :: Software Development :: Embedded Systems",
-        "Topic :: System :: Software Distribution",
-    ],
-    entry_points={"console_scripts": ["circup=circup:main"]},
-    # What does your project relate to?
-    keywords="adafruit, blinka, circuitpython, micropython, libraries",
-    # You can just specify the packages manually here if your project is
-    # simple. Or you can use find_packages().
-    packages=["circup"],
-    package_data={"circup": ["config/bundle_config.json"]},
-)

tests/import_styles.py

@@ -2,7 +2,10 @@
 #
 # SPDX-License-Identifier: MIT
 # pylint: disable=all
+import os, sys
 import adafruit_bus_device
 from adafruit_button import Button
 from adafruit_esp32spi import adafruit_esp32spi_socketpool
+from adafruit_display_text import wrap_text_to_pixels, wrap_text_to_lines
 import adafruit_hid.consumer_control
+import import_styles_sub

tests/mock_device/apps/test_app/import_styles.py

@@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: 2021 Jeff Epler for Adafruit Industries
+#
+# SPDX-License-Identifier: MIT
+# pylint: disable=all
+import os, sys
+import adafruit_bus_device
+from adafruit_button import Button
+from adafruit_esp32spi import adafruit_esp32spi_socketpool
+from adafruit_display_text import wrap_text_to_pixels, wrap_text_to_lines
+import adafruit_hid.consumer_control
+import import_styles_sub

tests/mock_device/apps/test_app/import_styles_sub.py

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: 2025 Neradoc
+#
+# SPDX-License-Identifier: MIT
+# pylint: disable=all
+import adafruit_ntp

tests/mock_device/boot_out.txt

@@ -1,3 +1,3 @@
-Adafruit CircuitPython 8.1.0 on 2019-08-02; Adafruit CircuitPlayground Express with samd21g18
+Adafruit CircuitPython 9.0.0 on 2019-08-02; Adafruit CircuitPlayground Express with samd21g18
 Board ID:this_is_a_board
 UID:AAAABBBBCCCC

tests/mock_device/import_styles.py

@@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: 2021 Jeff Epler for Adafruit Industries
+#
+# SPDX-License-Identifier: MIT
+# pylint: disable=all
+import os, sys
+import adafruit_bus_device
+from adafruit_button import Button
+from adafruit_esp32spi import adafruit_esp32spi_socketpool
+from adafruit_display_text import wrap_text_to_pixels, wrap_text_to_lines
+import adafruit_hid.consumer_control
+import import_styles_sub

tests/mock_device/import_styles_sub.py

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: 2025 Neradoc
+#
+# SPDX-License-Identifier: MIT
+# pylint: disable=all
+import adafruit_ntp

tests/mock_device_2/.gitignore

@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: 2025 Neradoc
+#
+# SPDX-License-Identifier: MIT
+lib/*

tests/mock_device_2/boot_out.txt

@@ -0,0 +1,3 @@
+Adafruit CircuitPython 9.0.0 on 2019-08-02; Adafruit CircuitPlayground Express with samd21g18
+Board ID:this_is_a_board
+UID:AAAABBBBCCCC

tests/mock_device_2/boot_out.txt.license

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2023 Tim Cocks, written for Adafruit Industries
+#
+# SPDX-License-Identifier: MIT

tests/mock_device_2/code.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2021 Jeff Epler for Adafruit Industries
+#
+# SPDX-License-Identifier: MIT
+# pylint: disable=all
+import adafruit_ssd1675
+import import_styles_sub
+import package

tests/mock_device_2/import_styles_sub.py

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: 2025 Neradoc
+#
+# SPDX-License-Identifier: MIT
+# pylint: disable=all
+import adafruit_ntp

tests/mock_device_2/package/__init__.py

@@ -0,0 +1,6 @@
+# SPDX-FileCopyrightText: 2025 Neradoc
+#
+# SPDX-License-Identifier: MIT
+# pylint: disable=all
+import adafruit_spd1656
+from .other import variable

tests/mock_device_2/package/other.py

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: 2021 Jeff Epler for Adafruit Industries
+#
+# SPDX-License-Identifier: MIT
+# pylint: disable=all
+import adafruit_spd1608

tests/test_circup.py

@@ -42,6 +42,9 @@ from circup.command_utils import (
     ensure_latest_bundle,
     get_bundle,
     get_bundles_dict,
+    imports_from_code,
+    get_all_imports,
+    libraries_from_auto_file,
 )
 from circup.shared import PLATFORMS
 from circup.module import Module
@@ -94,10 +97,10 @@ def test_Bundle_lib_dir():
             "adafruit/adafruit-circuitpython-bundle-py/"
             "adafruit-circuitpython-bundle-py-TESTTAG/lib"
         )
-        assert bundle.lib_dir("8mpy") == (
+        assert bundle.lib_dir("9mpy") == (
             circup.shared.DATA_DIR + "/"
-            "adafruit/adafruit-circuitpython-bundle-8mpy/"
-            "adafruit-circuitpython-bundle-8.x-mpy-TESTTAG/lib"
+            "adafruit/adafruit-circuitpython-bundle-9mpy/"
+            "adafruit-circuitpython-bundle-9.x-mpy-TESTTAG/lib"
         )
 
 
@@ -119,7 +122,7 @@ def test_get_bundles_dict():
     """
     with mock.patch(
         "circup.command_utils.BUNDLE_CONFIG_FILE", TEST_BUNDLE_CONFIG_JSON
-    ), mock.patch("circup.shared.BUNDLE_CONFIG_LOCAL", ""):
+    ), mock.patch("circup.command_utils.BUNDLE_CONFIG_LOCAL", ""):
         bundles_dict = get_bundles_dict()
         assert bundles_dict == TEST_BUNDLE_DATA
 
@@ -378,6 +381,16 @@ def test_Module_mpy_mismatch():
         assert m2.outofdate is False
         assert m3.mpy_mismatch is True
         assert m3.outofdate is True
+    with mock.patch(
+        "circup.backends.DiskBackend.get_circuitpython_version",
+        return_value=("9.0.0", ""),
+    ):
+        assert m1.mpy_mismatch is False
+        assert m1.outofdate is False
+        assert m2.mpy_mismatch is True
+        assert m2.outofdate is True
+        assert m3.mpy_mismatch is True
+        assert m3.outofdate is True
 
 
 def test_Module_major_update_bad_versions():
@@ -419,14 +432,16 @@ def test_Module_row():
     repo = "https://github.com/adafruit/SomeLibrary.git"
     with mock.patch("circup.os.path.isfile", return_value=True), mock.patch(
         "circup.backends.DiskBackend.get_circuitpython_version",
-        return_value=("8.0.0", ""),
+        return_value=("9.0.0", ""),
     ), mock.patch("circup.logger.warning") as mock_logger:
         backend = DiskBackend("mock_device", mock_logger)
         m = Module(name, backend, repo, "1.2.3", None, False, bundle, (None, None))
         assert m.row == ("module", "1.2.3", "unknown", "Major Version")
         m = Module(name, backend, repo, "1.2.3", "1.3.4", False, bundle, (None, None))
         assert m.row == ("module", "1.2.3", "1.3.4", "Minor Version")
-        m = Module(name, backend, repo, "1.2.3", "1.2.3", True, bundle, ("9.0.0", None))
+        m = Module(
+            name, backend, repo, "1.2.3", "1.2.3", True, bundle, ("8.0.0", "9.0.0")
+        )
         assert m.row == ("module", "1.2.3", "1.2.3", "MPY Format")
 
 
@@ -806,7 +821,7 @@ def test_get_circuitpython_version():
     with mock.patch("circup.logger.warning") as mock_logger:
         backend = DiskBackend("tests/mock_device", mock_logger)
         assert backend.get_circuitpython_version() == (
-            "8.1.0",
+            "9.0.0",
             "this_is_a_board",
         )
 
@@ -1117,8 +1132,33 @@ def test_show_match_py_command():
     assert "0 shown" in result.output
 
 
-def test_libraries_from_imports():
+def test_imports_from_code():
     """Ensure that various styles of import all work"""
+    test_file = str(pathlib.Path(__file__).parent / "import_styles.py")
+    with open(test_file, "r", encoding="utf8") as fp:
+        test_data = fp.read()
+
+    result = imports_from_code(test_data)
+    print(result)
+    assert result == [
+        "adafruit_bus_device",
+        "adafruit_button",
+        "adafruit_button.Button",
+        "adafruit_display_text",
+        "adafruit_display_text.wrap_text_to_lines",
+        "adafruit_display_text.wrap_text_to_pixels",
+        "adafruit_esp32spi",
+        "adafruit_esp32spi.adafruit_esp32spi_socketpool",
+        "adafruit_hid",
+        "adafruit_hid.consumer_control",
+        "import_styles_sub",
+        "os",
+        "sys",
+    ]
+
+
+def test_get_all_imports():
+    """List all libraries from auto file recursively"""
     mod_names = [
         "adafruit_bus_device",
         "adafruit_button",
@@ -1129,20 +1169,138 @@ def test_libraries_from_imports():
         "adafruit_oauth2",
         "adafruit_requests",
         "adafruit_touchscreen",
+        "adafruit_ntp",
     ]
-    test_file = str(pathlib.Path(__file__).parent / "import_styles.py")
 
-    result = circup.libraries_from_code_py(test_file, mod_names)
+    with mock.patch("circup.logger.info") as mock_logger, mock.patch(
+        "circup.os.path.isfile", return_value=True
+    ), mock.patch(
+        "circup.bundle.Bundle.lib_dir",
+        return_value="tests",
+    ):
+        tests_dir = pathlib.Path(__file__).parent
+        backend = DiskBackend(tests_dir / "mock_device", mock_logger)
+
+        test_file = str(tests_dir / "import_styles.py")
+        with open(test_file, "r", encoding="utf8") as fp:
+            test_data = fp.read()
+
+        result = get_all_imports(
+            backend,
+            test_data,
+            os.path.join(backend.device_location, "import_styles.py"),
+            mod_names,
+            current_module="",
+        )
+
     assert result == [
         "adafruit_bus_device",
         "adafruit_button",
+        "adafruit_display_text",
         "adafruit_esp32spi",
         "adafruit_hid",
+        "adafruit_ntp",
     ]
 
 
-def test_libraries_from_imports_bad():
-    """Ensure that we catch an import error"""
+def test_libraries_from_auto_file_local():
+    """Check that we get all libraries from auto file argument.
+    Testing here with a local file"""
+    mod_names = [
+        "adafruit_bus_device",
+        "adafruit_button",
+        "adafruit_display_shapes",
+        "adafruit_display_text",
+        "adafruit_esp32spi",
+        "adafruit_hid",
+        "adafruit_oauth2",
+        "adafruit_requests",
+        "adafruit_touchscreen",
+        "adafruit_ntp",
+    ]
+
+    auto_file = "apps/test_app/import_styles.py"
+
+    with mock.patch("circup.logger.info") as mock_logger, mock.patch(
+        "circup.os.path.isfile", return_value=True
+    ), mock.patch(
+        "circup.bundle.Bundle.lib_dir",
+        return_value="tests",
+    ):
+        tests_dir = pathlib.Path(__file__).parent
+        backend = DiskBackend(tests_dir / "mock_device", mock_logger)
+
+        result = libraries_from_auto_file(backend, auto_file, mod_names)
+
+    assert result == [
+        "adafruit_bus_device",
+        "adafruit_button",
+        "adafruit_display_text",
+        "adafruit_esp32spi",
+        "adafruit_hid",
+        "adafruit_ntp",
+    ]
+
+
+def test_libraries_from_auto_file_board():
+    """Check that we find code.py on the board if we give no auto_file argument"""
+    mod_names = [
+        "adafruit_bus_device",
+        "adafruit_button",
+        "adafruit_display_shapes",
+        "adafruit_display_text",
+        "adafruit_esp32spi",
+        "adafruit_ssd1675",
+        "adafruit_spd1656",
+        "adafruit_spd1608",
+        "adafruit_touchscreen",
+        "adafruit_ntp",
+    ]
+
+    auto_file = None
+
+    with mock.patch("circup.logger.info") as mock_logger, mock.patch(
+        "circup.os.path.isfile", return_value=True
+    ), mock.patch(
+        "circup.bundle.Bundle.lib_dir",
+        return_value="tests",
+    ):
+        tests_dir = pathlib.Path(__file__).parent
+        backend = DiskBackend(tests_dir / "mock_device_2", mock_logger)
+
+        result = libraries_from_auto_file(backend, auto_file, mod_names)
+
+    assert result == [
+        "adafruit_ntp",
+        "adafruit_spd1608",
+        "adafruit_spd1656",
+        "adafruit_ssd1675",
+    ]
+
+
+def test_libraries_from_auto_file_none():
+    """Check that we exit if we give no auto_file argument
+    and there's no default code file"""
+    mod_names = []
+    auto_file = None
+
+    with mock.patch("circup.logger.info") as mock_logger, mock.patch(
+        "circup.os.path.isfile", return_value=True
+    ), mock.patch(
+        "circup.bundle.Bundle.lib_dir",
+        return_value="tests",
+    ):
+        tests_dir = pathlib.Path(__file__).parent
+        backend = DiskBackend(tests_dir / "mock_device", mock_logger)
+        try:
+            libraries_from_auto_file(backend, auto_file, mod_names)
+            raise Exception("Did not call exit")
+        except SystemExit as ex:
+            assert ex.code == 1
+
+
+def test_install_auto_file_bad():
+    """Ensure that we catch an error when parsing auto file"""
     TEST_BUNDLE_MODULES = {"one.py": {}, "two.py": {}, "three.py": {}}
     runner = CliRunner()