Add commands for a local list of 3rd party bundles (#128)
* Add local bundle commands - the local bundles take precedence over the built-in ones - bundle-show lists the bundles --modules lists the modules in each bundle - bundle-add adds a bundle from the user/repo github string it does some level of checking that the repo's and zips URLs exist - bundle-remove removes a bundle from the local list * allow overwritting the built-in modules manually * remove show_bundles_info * filter github URLs into github repo string * avoid duplicates if adding built-ins as local, but allow it (to override priority order) * put bundle-remove --reset back * small change in bundle-show, local first * message on built-in in bundle-remove
This commit is contained in:
Neradoc
GitHub
parent
d467d2545b
commit
05514e5d79
4 changed files with 291 additions and 18 deletions
|
|
@ -34,6 +34,10 @@ DATA_DIR = appdirs.user_data_dir(appname="circup", appauthor="adafruit")
|
|||
BUNDLE_CONFIG_FILE = pkg_resources.resource_filename(
|
||||
"circup", "config/bundle_config.json"
|
||||
)
|
||||
#: Overwrite the bundles list with this file (only done manually)
|
||||
BUNDLE_CONFIG_OVERWRITE = os.path.join(DATA_DIR, "bundle_config.json")
|
||||
#: The path to the JSON file containing the local list of bundles.
|
||||
BUNDLE_CONFIG_LOCAL = os.path.join(DATA_DIR, "bundle_config_local.json")
|
||||
#: The path to the JSON file containing the metadata about the bundles.
|
||||
BUNDLE_DATA = os.path.join(DATA_DIR, "circup.json")
|
||||
#: The directory containing the utility's log file.
|
||||
|
|
@ -53,7 +57,7 @@ CPY_VERSION = ""
|
|||
#: Module formats list (and the other form used in github files)
|
||||
PLATFORMS = {"py": "py", "6mpy": "6.x-mpy", "7mpy": "7.x-mpy"}
|
||||
#: Commands that do not require an attached board
|
||||
BOARDLESS_COMMANDS = ["show"]
|
||||
BOARDLESS_COMMANDS = ["show", "bundle-add", "bundle-remove", "bundle-show"]
|
||||
|
||||
# Ensure DATA_DIR / LOG_DIR related directories and files exist.
|
||||
if not os.path.exists(DATA_DIR): # pragma: no cover
|
||||
|
|
@ -170,6 +174,26 @@ class Bundle:
|
|||
self._latest = get_latest_release_from_url(self.url + "/releases/latest")
|
||||
return self._latest
|
||||
|
||||
def validate(self):
|
||||
"""
|
||||
Test the existence of the expected URLs (not their content)
|
||||
"""
|
||||
tag = self.latest_tag
|
||||
if not tag or tag == "releases":
|
||||
if VERBOSE:
|
||||
click.secho(f' Invalid tag "{tag}"', fg="red")
|
||||
return False
|
||||
for platform in PLATFORMS.values():
|
||||
url = self.url_format.format(platform=platform, tag=tag)
|
||||
r = requests.get(url, stream=True)
|
||||
# pylint: disable=no-member
|
||||
if r.status_code != requests.codes.ok:
|
||||
if VERBOSE:
|
||||
click.secho(f" Unable to find {os.path.split(url)[1]}", fg="red")
|
||||
return False
|
||||
# pylint: enable=no-member
|
||||
return True
|
||||
|
||||
def __repr__(self):
|
||||
"""
|
||||
Helps with log files.
|
||||
|
|
@ -702,14 +726,51 @@ def get_bundle_versions(bundles_list, avoid_download=False):
|
|||
return all_the_modules
|
||||
|
||||
|
||||
def get_bundles_dict():
|
||||
"""
|
||||
Retrieve the dictionary from BUNDLE_CONFIG_FILE (JSON).
|
||||
Put the local dictionary in front, so it gets priority.
|
||||
It's a dictionary of bundle string identifiers.
|
||||
|
||||
:return: Combined dictionaries from the config files.
|
||||
"""
|
||||
bundle_dict = get_bundles_local_dict()
|
||||
try:
|
||||
with open(BUNDLE_CONFIG_OVERWRITE) as bundle_config_json:
|
||||
bundle_config = json.load(bundle_config_json)
|
||||
except (FileNotFoundError, json.decoder.JSONDecodeError):
|
||||
with open(BUNDLE_CONFIG_FILE) as bundle_config_json:
|
||||
bundle_config = json.load(bundle_config_json)
|
||||
for name, bundle in bundle_config.items():
|
||||
if bundle not in bundle_dict.values():
|
||||
bundle_dict[name] = bundle
|
||||
return bundle_dict
|
||||
|
||||
|
||||
def get_bundles_local_dict():
|
||||
"""
|
||||
Retrieve the local bundles from BUNDLE_CONFIG_LOCAL (JSON).
|
||||
|
||||
:return: Raw dictionary from the config file(s).
|
||||
"""
|
||||
try:
|
||||
with open(BUNDLE_CONFIG_LOCAL) as bundle_config_json:
|
||||
bundle_config = json.load(bundle_config_json)
|
||||
if not isinstance(bundle_config, dict) or not bundle_config:
|
||||
logger.error("Local bundle list invalid. Skipped.")
|
||||
raise FileNotFoundError("Bad local bundle list")
|
||||
return bundle_config
|
||||
except (FileNotFoundError, json.decoder.JSONDecodeError):
|
||||
return dict()
|
||||
|
||||
|
||||
def get_bundles_list():
|
||||
"""
|
||||
Retrieve the list of bundles as listed BUNDLE_CONFIG_FILE (JSON)
|
||||
Retrieve the list of bundles from the config dictionary.
|
||||
|
||||
:return: List of supported bundles as Bundle objects.
|
||||
"""
|
||||
with open(BUNDLE_CONFIG_FILE) as bundle_config_json:
|
||||
bundle_config = json.load(bundle_config_json)
|
||||
bundle_config = get_bundles_dict()
|
||||
bundles_list = [Bundle(bundle_config[b]) for b in bundle_config]
|
||||
logger.info("Using bundles: %s", ", ".join(b.key for b in bundles_list))
|
||||
return bundles_list
|
||||
|
|
@ -979,6 +1040,20 @@ def libraries_from_requirements(requirements):
|
|||
return libraries
|
||||
|
||||
|
||||
def save_local_bundles(bundles_data):
|
||||
"""
|
||||
Save the list of local bundles to the settings.
|
||||
|
||||
:param str key: The bundle's identifier/key.
|
||||
"""
|
||||
if len(bundles_data) > 0:
|
||||
with open(BUNDLE_CONFIG_LOCAL, "w", encoding="utf-8") as data:
|
||||
json.dump(bundles_data, data)
|
||||
else:
|
||||
if os.path.isfile(BUNDLE_CONFIG_LOCAL):
|
||||
os.unlink(BUNDLE_CONFIG_LOCAL)
|
||||
|
||||
|
||||
def tags_data_load():
|
||||
"""
|
||||
Load the list of the version tags of the bundles on disk.
|
||||
|
|
@ -1053,6 +1128,9 @@ def main(ctx, verbose, path): # pragma: no cover
|
|||
logger.addHandler(verbose_handler)
|
||||
click.echo("Logging to {}\n".format(LOGFILE))
|
||||
logger.info("### Started Circup ###")
|
||||
# stop early if the command is boardless
|
||||
if ctx.invoked_subcommand in BOARDLESS_COMMANDS:
|
||||
return
|
||||
if path:
|
||||
device_path = path
|
||||
else:
|
||||
|
|
@ -1062,14 +1140,7 @@ def main(ctx, verbose, path): # pragma: no cover
|
|||
"https://github.com/adafruit/circuitpython/releases/latest"
|
||||
)
|
||||
global CPY_VERSION
|
||||
if device_path is None and ctx.invoked_subcommand in BOARDLESS_COMMANDS:
|
||||
CPY_VERSION = latest_version
|
||||
click.echo(
|
||||
"No CircuitPython device detected. Using CircuitPython {}.".format(
|
||||
CPY_VERSION
|
||||
)
|
||||
)
|
||||
elif device_path is None:
|
||||
if device_path is None:
|
||||
click.secho("Could not find a connected CircuitPython device.", fg="red")
|
||||
sys.exit(1)
|
||||
else:
|
||||
|
|
@ -1125,9 +1196,9 @@ def freeze(ctx, requirement): # pragma: no cover
|
|||
click.echo("No modules found on the device.")
|
||||
|
||||
|
||||
@main.command()
|
||||
@main.command("list")
|
||||
@click.pass_context
|
||||
def list(ctx): # pragma: no cover
|
||||
def list_cli(ctx): # pragma: no cover
|
||||
"""
|
||||
Lists all out of date modules found on the connected CIRCUITPYTHON device.
|
||||
"""
|
||||
|
|
@ -1218,8 +1289,8 @@ def install(ctx, modules, py, requirement, auto, auto_file): # pragma: no cover
|
|||
# pylint: enable=too-many-arguments,too-many-locals
|
||||
|
||||
|
||||
@click.argument("match", required=False, nargs=1)
|
||||
@main.command()
|
||||
@click.argument("match", required=False, nargs=1)
|
||||
def show(match): # pragma: no cover
|
||||
"""
|
||||
Show a list of available modules in the bundle. These are modules which
|
||||
|
|
@ -1350,6 +1421,123 @@ def update(ctx, all): # pragma: no cover
|
|||
click.echo("None of the modules found on the device need an update.")
|
||||
|
||||
|
||||
@main.command("bundle-show")
|
||||
@click.option("--modules", is_flag=True, help="List all the modules per bundle.")
|
||||
def bundle_show(modules):
|
||||
"""
|
||||
Show the list of bundles, default and local, with URL, current version
|
||||
and latest version retrieved from the web.
|
||||
"""
|
||||
locals = get_bundles_local_dict().values()
|
||||
bundles = get_bundles_list()
|
||||
available_modules = get_bundle_versions(bundles)
|
||||
|
||||
for bundle in bundles:
|
||||
if bundle.key in locals:
|
||||
click.secho(bundle.key, fg="yellow")
|
||||
else:
|
||||
click.secho(bundle.key, fg="green")
|
||||
click.echo(" " + bundle.url)
|
||||
click.echo(" version = " + bundle.current_tag)
|
||||
if modules:
|
||||
click.echo("Modules:")
|
||||
for name, mod in sorted(available_modules.items()):
|
||||
if mod["bundle"] == bundle:
|
||||
click.echo(f" {name} ({mod.get('__version__', '-')})")
|
||||
|
||||
|
||||
@main.command("bundle-add")
|
||||
@click.argument("bundle", nargs=-1)
|
||||
def bundle_add(bundle):
|
||||
"""
|
||||
Add bundles to the local bundles list, by "user/repo" github string.
|
||||
A series of tests to validate that the bundle exists and at least looks
|
||||
like a bundle are done before validating it. There might still be errors
|
||||
when the bundle is downloaded for the first time.
|
||||
"""
|
||||
bundles_dict = get_bundles_local_dict()
|
||||
modified = False
|
||||
for bun in bundle:
|
||||
# cleanup in case seombody pastes the URL to the repo/releases
|
||||
bun = re.sub(r"https?://github.com/([^/]+/[^/]+)(/.*)?", r"\1", bun)
|
||||
if bun in bundles_dict.values():
|
||||
click.secho("Bundle already in list.", fg="yellow")
|
||||
click.secho(" " + bun, fg="yellow")
|
||||
continue
|
||||
try:
|
||||
bb = Bundle(bun)
|
||||
except ValueError:
|
||||
click.secho(
|
||||
"Bundle string invalid, expecting github URL or `user/repository` string.",
|
||||
fg="red",
|
||||
)
|
||||
click.secho(" " + bun, fg="red")
|
||||
continue
|
||||
result = requests.get("https://github.com/" + bun)
|
||||
# pylint: disable=no-member
|
||||
if result.status_code == requests.codes.NOT_FOUND:
|
||||
click.secho("Bundle invalid, the repository doesn't exist (404).", fg="red")
|
||||
click.secho(" " + bun, fg="red")
|
||||
continue
|
||||
# pylint: enable=no-member
|
||||
if not bb.validate():
|
||||
click.secho(
|
||||
"Bundle invalid, is the repository a valid circup bundle ?", fg="red"
|
||||
)
|
||||
click.secho(" " + bun, fg="red")
|
||||
continue
|
||||
# note: use bun as the dictionary key for uniqueness
|
||||
bundles_dict[bun] = bun
|
||||
modified = True
|
||||
click.echo("Added " + bun)
|
||||
click.echo(" " + bb.url)
|
||||
if modified:
|
||||
# save the bundles list
|
||||
save_local_bundles(bundles_dict)
|
||||
# update and get the new bundles for the first time
|
||||
get_bundle_versions(get_bundles_list())
|
||||
|
||||
|
||||
@main.command("bundle-remove")
|
||||
@click.argument("bundle", nargs=-1)
|
||||
@click.option("--reset", is_flag=True, help="Remove all local bundles.")
|
||||
def bundle_remove(bundle, reset):
|
||||
"""
|
||||
Remove one or more bundles from the local bundles list.
|
||||
"""
|
||||
if reset:
|
||||
save_local_bundles({})
|
||||
return
|
||||
bundle_config = list(get_bundles_dict().values())
|
||||
bundles_local_dict = get_bundles_local_dict()
|
||||
modified = False
|
||||
for bun in bundle:
|
||||
# cleanup in case seombody pastes the URL to the repo/releases
|
||||
bun = re.sub(r"https?://github.com/([^/]+/[^/]+)(/.*)?", r"\1", bun)
|
||||
found = False
|
||||
for name, repo in list(bundles_local_dict.items()):
|
||||
if bun in (name, repo):
|
||||
found = True
|
||||
click.secho(f"Bundle {repo}")
|
||||
do_it = click.confirm("Do you want to remove that bundle ?")
|
||||
if do_it:
|
||||
click.secho("Removing the bundle from the local list", fg="yellow")
|
||||
click.secho(f" {bun}", fg="yellow")
|
||||
modified = True
|
||||
del bundles_local_dict[name]
|
||||
if not found:
|
||||
if bun in bundle_config:
|
||||
click.secho("Cannot remove built-in module:" "\n " + bun, fg="red")
|
||||
else:
|
||||
click.secho(
|
||||
"Bundle not found in the local list, nothing removed:"
|
||||
"\n " + bun,
|
||||
fg="red",
|
||||
)
|
||||
if modified:
|
||||
save_local_bundles(bundles_local_dict)
|
||||
|
||||
|
||||
# Allows execution via `python -m circup ...`
|
||||
# pylint: disable=no-value-for-parameter
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
|
|
|
|||
3
tests/test_bundle_config_local.json
Normal file
3
tests/test_bundle_config_local.json
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"local_bundle": "Neradoc/Circuitpython_Keyboard_Layouts"
|
||||
}
|
||||
3
tests/test_bundle_config_local.json.license
Normal file
3
tests/test_bundle_config_local.json.license
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
# SPDX-FileCopyrightText: 2021 Neradoc NeraOnGit@ri1.fr
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
|
@ -40,8 +40,12 @@ import circup
|
|||
|
||||
TEST_BUNDLE_CONFIG_JSON = "tests/test_bundle_config.json"
|
||||
with open(TEST_BUNDLE_CONFIG_JSON) as tbc:
|
||||
test_bundle_data = json.load(tbc)
|
||||
TEST_BUNDLE_NAME = test_bundle_data["test_bundle"]
|
||||
TEST_BUNDLE_DATA = json.load(tbc)
|
||||
TEST_BUNDLE_NAME = TEST_BUNDLE_DATA["test_bundle"]
|
||||
|
||||
TEST_BUNDLE_CONFIG_LOCAL_JSON = "tests/test_bundle_config_local.json"
|
||||
with open(TEST_BUNDLE_CONFIG_LOCAL_JSON) as tbc:
|
||||
TEST_BUNDLE_LOCAL_DATA = json.load(tbc)
|
||||
|
||||
|
||||
def test_Bundle_init():
|
||||
|
|
@ -120,16 +124,91 @@ def test_Bundle_latest_tag():
|
|||
assert bundle.latest_tag == "BESTESTTAG"
|
||||
|
||||
|
||||
def test_get_bundles_dict():
|
||||
"""
|
||||
Check we are getting the bundles list from BUNDLE_CONFIG_FILE.
|
||||
"""
|
||||
with mock.patch("circup.BUNDLE_CONFIG_FILE", TEST_BUNDLE_CONFIG_JSON), mock.patch(
|
||||
"circup.BUNDLE_CONFIG_LOCAL", ""
|
||||
):
|
||||
bundles_dict = circup.get_bundles_dict()
|
||||
assert bundles_dict == TEST_BUNDLE_DATA
|
||||
|
||||
with mock.patch("circup.BUNDLE_CONFIG_FILE", TEST_BUNDLE_CONFIG_JSON), mock.patch(
|
||||
"circup.BUNDLE_CONFIG_LOCAL", TEST_BUNDLE_CONFIG_LOCAL_JSON
|
||||
):
|
||||
bundles_dict = circup.get_bundles_dict()
|
||||
expected_dict = {**TEST_BUNDLE_LOCAL_DATA, **TEST_BUNDLE_DATA}
|
||||
assert bundles_dict == expected_dict
|
||||
|
||||
|
||||
def test_get_bundles_local_dict():
|
||||
"""
|
||||
Check we are getting the bundles list from BUNDLE_CONFIG_LOCAL.
|
||||
"""
|
||||
with mock.patch("circup.BUNDLE_CONFIG_FILE", TEST_BUNDLE_CONFIG_JSON), mock.patch(
|
||||
"circup.BUNDLE_CONFIG_LOCAL", ""
|
||||
):
|
||||
bundles_dict = circup.get_bundles_dict()
|
||||
assert bundles_dict == TEST_BUNDLE_DATA
|
||||
|
||||
with mock.patch("circup.BUNDLE_CONFIG_FILE", TEST_BUNDLE_CONFIG_JSON), mock.patch(
|
||||
"circup.BUNDLE_CONFIG_LOCAL", TEST_BUNDLE_CONFIG_LOCAL_JSON
|
||||
):
|
||||
bundles_dict = circup.get_bundles_dict()
|
||||
expected_dict = {**TEST_BUNDLE_LOCAL_DATA, **TEST_BUNDLE_DATA}
|
||||
assert bundles_dict == expected_dict
|
||||
|
||||
|
||||
def test_get_bundles_list():
|
||||
"""
|
||||
Check we are getting the bundles list from BUNDLE_CONFIG_FILE.
|
||||
"""
|
||||
with mock.patch("circup.BUNDLE_CONFIG_FILE", TEST_BUNDLE_CONFIG_JSON):
|
||||
with mock.patch("circup.BUNDLE_CONFIG_FILE", TEST_BUNDLE_CONFIG_JSON), mock.patch(
|
||||
"circup.BUNDLE_CONFIG_LOCAL", ""
|
||||
):
|
||||
bundles_list = circup.get_bundles_list()
|
||||
bundle = circup.Bundle(TEST_BUNDLE_NAME)
|
||||
assert repr(bundles_list) == repr([bundle])
|
||||
|
||||
|
||||
def test_save_local_bundles():
|
||||
"""
|
||||
Pretend to save local bundles.
|
||||
"""
|
||||
with mock.patch("circup.BUNDLE_CONFIG_FILE", TEST_BUNDLE_CONFIG_JSON), mock.patch(
|
||||
"circup.BUNDLE_CONFIG_LOCAL", ""
|
||||
), mock.patch("circup.os.unlink") as mock_unlink, mock.patch(
|
||||
"circup.json.dump"
|
||||
) as mock_dump, mock.patch(
|
||||
"circup.json.load", return_value=TEST_BUNDLE_DATA
|
||||
), mock.patch(
|
||||
"circup.open", mock.mock_open()
|
||||
) as mock_open:
|
||||
final_data = {**TEST_BUNDLE_DATA, **TEST_BUNDLE_LOCAL_DATA}
|
||||
circup.save_local_bundles(final_data)
|
||||
mock_dump.assert_called_once_with(final_data, mock_open())
|
||||
mock_unlink.assert_not_called()
|
||||
|
||||
|
||||
def test_save_local_bundles_reset():
|
||||
"""
|
||||
Pretend to reset the local bundles.
|
||||
"""
|
||||
with mock.patch("circup.BUNDLE_CONFIG_FILE", TEST_BUNDLE_CONFIG_JSON), mock.patch(
|
||||
"circup.BUNDLE_CONFIG_LOCAL", "test/NOTEXISTS"
|
||||
), mock.patch("circup.os.path.isfile", return_value=True), mock.patch(
|
||||
"circup.os.unlink"
|
||||
) as mock_unlink, mock.patch(
|
||||
"circup.json.load", return_value=TEST_BUNDLE_DATA
|
||||
), mock.patch(
|
||||
"circup.open", mock.mock_open()
|
||||
) as mock_open:
|
||||
circup.save_local_bundles({})
|
||||
mock_open().write.assert_not_called()
|
||||
mock_unlink.assert_called_once_with("test/NOTEXISTS")
|
||||
|
||||
|
||||
def test_Module_init_file_module():
|
||||
"""
|
||||
Ensure the Module instance is set up as expected and logged, as if for a
|
||||
|
|
|
|||
Loading…
Reference in a new issue