djsundog/circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tools/mpremote: Add mpremote mip install to install packages.

Browse Source 
This supports the same package sources as the new `mip` tool.
 - micropython-lib (by name)
 - http(s) & github packages with json description
 - directly downloading a .py/.mpy file

The version is specified with an optional `@version` on the end of the
package name. The target dir, index, and mpy/no-mpy can be set through
command line args.

This work was funded through GitHub Sponsors.

Signed-off-by: Jim Mussared <jim.mussared@gmail.com>
This commit is contained in:
Jim Mussared 2022-09-29 00:45:34 +10:00
parent 68d094358e
commit 12ca918eb2
6 changed files with 272 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
docs/reference/mpremote.rst
View File

@ -146,6 +146,14 @@ The full list of supported commands are:
variable ``$EDITOR``). If the editor exits successfully, the updated file will
be copied back to the device.
- install packages from :term:`micropython-lib` (or GitHub) using the ``mip`` tool:
.. code-block:: bash
$ mpremote mip install <packages...>
See :ref:`packages` for more information.
- mount the local directory on the remote device:
.. code-block:: bash
@ -269,3 +277,9 @@ Examples
mpremote cp -r dir/ :
mpremote cp a.py b.py : + repl
mpremote mip install aioble
mpremote mip install github:org/repo@branch
mpremote mip install --target /flash/third-party functools

16
docs/reference/packages.rst
View File

@ -78,17 +78,17 @@ The :term:`mpremote` tool also includes the same functionality as ``mip`` and
can be used from a host PC to install packages to a locally connected device
(e.g. via USB or UART)::
$ mpremote install pkgname
$ mpremote install pkgname@x.y
$ mpremote install http://example.com/x/y/foo.py
$ mpremote install github:org/repo
$ mpremote install github:org/repo@branch-or-tag
$ mpremote mip install pkgname
$ mpremote mip install pkgname@x.y
$ mpremote mip install http://example.com/x/y/foo.py
$ mpremote mip install github:org/repo
$ mpremote mip install github:org/repo@branch-or-tag
The ``--target=path``, ``--no-mpy``, and ``--index`` arguments can be set::
$ mpremote install --target=/flash/third-party pkgname
$ mpremote install --no-mpy pkgname
$ mpremote install --index https://host/pi pkgname
$ mpremote mip install --target=/flash/third-party pkgname
$ mpremote mip install --no-mpy pkgname
$ mpremote mip install --index https://host/pi pkgname
Installing packages manually
----------------------------

41
tools/mpremote/README.md
View File

@ -11,23 +11,28 @@ This will automatically connect to the device and provide an interactive REPL.
The full list of supported commands are:
mpremote connect <device> -- connect to given device
device may be: list, auto, id:x, port:x
or any valid device name/path
mpremote disconnect -- disconnect current device
mpremote mount <local-dir> -- mount local directory on device
mpremote eval <string> -- evaluate and print the string
mpremote exec <string> -- execute the string
mpremote run <file> -- run the given local script
mpremote fs <command> <args...> -- execute filesystem commands on the device
command may be: cat, ls, cp, rm, mkdir, rmdir
use ":" as a prefix to specify a file on the device
mpremote repl -- enter REPL
options:
--capture <file>
--inject-code <string>
--inject-file <file>
mpremote help -- print list of commands and exit
mpremote connect <device> -- connect to given device
device may be: list, auto, id:x, port:x
or any valid device name/path
mpremote disconnect -- disconnect current device
mpremote mount <local-dir> -- mount local directory on device
mpremote eval <string> -- evaluate and print the string
mpremote exec <string> -- execute the string
mpremote run <file> -- run the given local script
mpremote fs <command> <args...> -- execute filesystem commands on the device
command may be: cat, ls, cp, rm, mkdir, rmdir
use ":" as a prefix to specify a file on the device
mpremote repl -- enter REPL
options:
--capture <file>
--inject-code <string>
--inject-file <file>
mpremote mip install <package...> -- Install packages (from micropython-lib or third-party sources)
options:
--target <path>
--index <url>
--no-mpy
mpremote help -- print list of commands and exit
Multiple commands can be specified and they will be run sequentially. Connection
and disconnection will be done automatically at the start and end of the execution
@ -73,3 +78,5 @@ Examples:
mpremote cp :main.py .
mpremote cp main.py :
mpremote cp -r dir/ :
mpremote mip install aioble
mpremote mip install github:org/repo@branch

28
tools/mpremote/mpremote/main.py
View File

@ -36,6 +36,7 @@ from .commands import (
do_resume,
do_soft_reset,
)
from .mip import do_mip
from .repl import do_repl
_PROG = "mpremote"
@ -162,6 +163,29 @@ def argparse_filesystem():
return cmd_parser
def argparse_mip():
cmd_parser = argparse.ArgumentParser(
description="install packages from micropython-lib or third-party sources"
)
_bool_flag(cmd_parser, "mpy", "m", True, "download as compiled .mpy files (default)")
cmd_parser.add_argument(
"--target", type=str, required=False, help="destination direction on the device"
)
cmd_parser.add_argument(
"--index",
type=str,
required=False,
help="package index to use (defaults to micropython-lib)",
)
cmd_parser.add_argument("command", nargs=1, help="mip command (e.g. install)")
cmd_parser.add_argument(
"packages",
nargs="+",
help="list package specifications, e.g. name, name@version, github:org/repo, github:org/repo@branch",
)
return cmd_parser
def argparse_none(description):
return lambda: argparse.ArgumentParser(description=description)
@ -216,6 +240,10 @@ _COMMANDS = {
do_filesystem,
argparse_filesystem,
),
"mip": (
do_mip,
argparse_mip,
),
"help": (
do_help,
argparse_none("print help and exit"),

191
tools/mpremote/mpremote/mip.py Normal file
View File

@ -0,0 +1,191 @@
# Micropython package installer
# Ported from micropython-lib/micropython/mip/mip.py.
# MIT license; Copyright (c) 2022 Jim Mussared
import urllib.error
import urllib.request
import json
import tempfile
import os
from .commands import CommandError, show_progress_bar
_PACKAGE_INDEX = "https://micropython.org/pi/v2"
_CHUNK_SIZE = 128
# This implements os.makedirs(os.dirname(path))
def _ensure_path_exists(pyb, path):
import os
split = path.split("/")
# Handle paths starting with "/".
if not split[0]:
split.pop(0)
split[0] = "/" + split[0]
prefix = ""
for i in range(len(split) - 1):
prefix += split[i]
if not pyb.fs_exists(prefix):
pyb.fs_mkdir(prefix)
prefix += "/"
# Copy from src (stream) to dest (function-taking-bytes)
def _chunk(src, dest, length=None, op="downloading"):
buf = memoryview(bytearray(_CHUNK_SIZE))
total = 0
if length:
show_progress_bar(0, length, op)
while True:
n = src.readinto(buf)
if n == 0:
break
dest(buf if n == _CHUNK_SIZE else buf[:n])
total += n
if length:
show_progress_bar(total, length, op)
def _rewrite_url(url, branch=None):
if not branch:
branch = "HEAD"
if url.startswith("github:"):
url = url[7:].split("/")
url = (
"https://raw.githubusercontent.com/"
+ url[0]
+ "/"
+ url[1]
+ "/"
+ branch
+ "/"
+ "/".join(url[2:])
)
return url
def _download_file(pyb, url, dest):
try:
with urllib.request.urlopen(url) as src:
fd, path = tempfile.mkstemp()
try:
print("Installing:", dest)
with os.fdopen(fd, "wb") as f:
_chunk(src, f.write, src.length)
_ensure_path_exists(pyb, dest)
pyb.fs_put(path, dest, progress_callback=show_progress_bar)
finally:
os.unlink(path)
except urllib.error.HTTPError as e:
if e.status == 404:
raise CommandError(f"File not found: {url}")
else:
raise CommandError(f"Error {e.status} requesting {url}")
except urllib.error.URLError as e:
raise CommandError(f"{e.reason} requesting {url}")
def _install_json(pyb, package_json_url, index, target, version, mpy):
try:
with urllib.request.urlopen(_rewrite_url(package_json_url, version)) as response:
package_json = json.load(response)
except urllib.error.HTTPError as e:
if e.status == 404:
raise CommandError(f"Package not found: {package_json_url}")
else:
raise CommandError(f"Error {e.status} requesting {package_json_url}")
except urllib.error.URLError as e:
raise CommandError(f"{e.reason} requesting {package_json_url}")
for target_path, short_hash in package_json.get("hashes", ()):
fs_target_path = target + "/" + target_path
file_url = f"{index}/file/{short_hash[:2]}/{short_hash}"
_download_file(pyb, file_url, fs_target_path)
for target_path, url in package_json.get("urls", ()):
fs_target_path = target + "/" + target_path
_download_file(pyb, _rewrite_url(url, version), fs_target_path)
for dep, dep_version in package_json.get("deps", ()):
_install_package(pyb, dep, index, target, dep_version, mpy)
def _install_package(pyb, package, index, target, version, mpy):
if (
package.startswith("http://")
or package.startswith("https://")
or package.startswith("github:")
):
if package.endswith(".py") or package.endswith(".mpy"):
print(f"Downloading {package} to {target}")
_download_file(
pyb, _rewrite_url(package, version), target + "/" + package.rsplit("/")[-1]
)
return
else:
if not package.endswith(".json"):
if not package.endswith("/"):
package += "/"
package += "package.json"
print(f"Installing {package} to {target}")
else:
if not version:
version = "latest"
print(f"Installing {package} ({version}) from {index} to {target}")
mpy_version = "py"
if mpy:
pyb.exec("import sys")
mpy_version = (
int(pyb.eval("getattr(sys.implementation, '_mpy', 0) & 0xFF").decode()) or "py"
)
package = f"{index}/package/{mpy_version}/{package}/{version}.json"
_install_json(pyb, package, index, target, version, mpy)
def do_mip(state, args):
state.did_action()
if args.command[0] == "install":
state.ensure_raw_repl()
for package in args.packages:
version = None
if "@" in package:
package, version = package.split("@")
print("Install", package)
if args.index is None:
args.index = _PACKAGE_INDEX
if args.target is None:
state.pyb.exec("import sys")
lib_paths = (
state.pyb.eval("'\\n'.join(p for p in sys.path if p.endswith('/lib'))")
.decode()
.split("\n")
)
if lib_paths and lib_paths[0]:
args.target = lib_paths[0]
else:
raise CommandError(
"Unable to find lib dir in sys.path, use --target to override"
)
if args.mpy is None:
args.mpy = True
try:
_install_package(
state.pyb, package, args.index.rstrip("/"), args.target, version, args.mpy
)
except CommandError:
print("Package may be partially installed")
raise
print("Done")
else:
raise CommandError(f"mip: '{args.command[0]}' is not a command")

7
tools/pyboard.py
View File

@ -476,6 +476,13 @@ class Pyboard:
t = str(self.eval("pyb.RTC().datetime()"), encoding="utf8")[1:-1].split(", ")
return int(t[4]) * 3600 + int(t[5]) * 60 + int(t[6])
def fs_exists(self, src):
try:
self.exec_("import uos\nuos.stat(%s)" % (("'%s'" % src) if src else ""))
return True
except PyboardError:
return False
def fs_ls(self, src):
cmd = (
"import uos\nfor f in uos.ilistdir(%s):\n"