From 41039445c9440d87fa0c232030e91eee1d7f06f9 Mon Sep 17 00:00:00 2001 From: Scott Shawcroft Date: Fri, 24 Jun 2022 17:02:11 -0700 Subject: [PATCH] Clean up and add docs --- docs/index.rst | 1 + docs/workflows.md | 267 ++++++++++++++++++ .../espressif/common-hal/mdns/RemoteService.c | 6 +- ports/espressif/common-hal/mdns/Server.c | 1 - .../espressif/common-hal/socketpool/Socket.c | 14 +- ports/espressif/common-hal/wifi/Radio.c | 21 +- ports/espressif/common-hal/wifi/__init__.c | 1 - .../esp-idf-config/sdkconfig.defaults | 2 +- shared-bindings/mdns/RemoteService.c | 13 + shared-bindings/mdns/RemoteService.h | 4 +- supervisor/shared/web_workflow/web_workflow.c | 2 +- 11 files changed, 296 insertions(+), 36 deletions(-) create mode 100644 docs/workflows.md diff --git a/docs/index.rst b/docs/index.rst index 6a67fe0b7c..a200a5238b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -22,6 +22,7 @@ Full Table of Contents supported_ports.rst troubleshooting.rst drivers.rst + workflows environment.rst .. toctree:: diff --git a/docs/workflows.md b/docs/workflows.md new file mode 100644 index 0000000000..74f35532e3 --- /dev/null +++ b/docs/workflows.md @@ -0,0 +1,267 @@ +# Workflows + +Workflows are the process used to 1) manipulate files on the CircuitPython device and 2) interact +with the serial connection to CircuitPython. The serial connection is usually used to access the +REPL. + +Starting with CircuitPython 3.x we moved to a USB-only workflow. Prior to that, we used the serial +connection alone to do the whole workflow. In CircuitPython 7.x, a BLE workflow was added with the +advantage of working with mobile devices. CircuitPython 8.x added a web workflow that works over the +local network (usually Wi-Fi) and a web browser. Other clients can also use the Web REST API. Boards +should clearly document which workflows are supported. + +Code for workflows lives in `supervisor/shared`. + +The workflow APIs are documented here. + +## USB + +These USB interfaces are enabled by default on boards with USB support. They are usable once the +device has been plugged into a host. + +### CIRCUITPY drive +CircuitPython exposes a standard mass storage (MSC) interface to enable file manipulation over a +standard interface. This interface works underneath the file system at the block level so using it +excludes other types of workflows from manipulating the file system at the same time. + +### CDC serial +CircuitPython exposes one CDC USB interface for CircuitPython serial. This is a standard serial +USB interface. + +TODO: Document how it designates itself from the user CDC. + +Setting baudrate 1200 and disconnecting will reboot into a bootloader. (Used by Arduino to trigger +a reset into bootloader.) + +## BLE + +The BLE workflow is enabled for nRF boards. By default, to prevent malicious access, it is disabled. +To connect to the BLE workflow, press the reset button while the status led blinks blue quickly +after the safe mode blinks. The board will restart and broadcast the file transfer service UUID +(`0xfebb`) along with the board's [Creation IDs](https://github.com/creationid/creators). This +public broadcast is done at a lower transmit level so the devices must be closer. On connection, the +device will need to pair and bond. Once bonded, the device will broadcast whenever disconnected +using a rotating key rather than a static one. Non-bonded devices won't be able to resolve it. After +connection, the central device can discover two default services. One for file transfer and one for +CircuitPython specifically that includes serial characteristics. + +### File Transfer API + +CircuitPython uses [an open File Transfer API](https://github.com/adafruit/Adafruit_CircuitPython_BLE_File_Transfer) +to enable file system access. + +### CircuitPython Service + +The base UUID for the CircuitPython service is `ADAFXXXX-4369-7263-7569-7450794686e`. The `XXXX` is +replaced by the four specific digits below. The service itself is `0001`. + +#### TX - `0002` / RX - `0003` + +These characteristic work just like the Nordic Uart Service (NUS) but have different UUIDs to prevent +conflicts with user created NUS services. + +#### Version - `0100` +Read-only characteristic that returns the UTF-8 encoded version string. + +## Web + +The web workflow is depends on adding Wi-Fi credentials into the `/.env` file. The keys are +`CIRCUITPY_WIFI_SSID` and `CIRCUITPY_WIFI_PASSWORD`. Once these are defined, CircuitPython will +automatically connect to the network and start the webserver used for the workflow. The webserver +is on port 80. It also enables MDNS. + +MDNS is used to resolve [`circuitpython.local`](http://circuitpython.local) to a device specific +hostname of the form `cpy-XXXXXX.local`. The `XXXXXX` is based on network MAC address. The device +also provides the MDNS service with service type `_circuitpython` and protocol `_tcp`. + +The web server is HTTP 1.1 and may use chunked responses so that it doesn't need to precompute +content length. + +### `/` +The root welcome page links to the file system page and also displays other CircuitPython devices +found using MDNS service discovery. This allows web browsers to find other devices from one. (All +devices will respond to `circuitpython.local` so the device redirected to may vary.) + +### CORS +The web server will allow requests from `cpy-XXXXXX.local`, `127.0.0.1`, the device's IP and +`code.circuitpython.org`. (`circuitpython.local` requests will be redirected to `cpy-XXXXXX.local`.) + +### File REST API +All file system related APIs are protected by HTTP basic authentication. It is *NOT* secure but will +hopefully prevent some griefing in shared settings. The password is sent unencrypted so do not reuse +a password with something important. + +The password is taken from `/.env` with the key `CIRCUITPY_WEB_API_PASSWORD`. If this is unset, the +server will respond with `403 Forbidden`. When a password is set, but not provided in a request, it +will respond `401 Unauthorized`. + +#### `/fs/` + +The `/fs/` page will respond with a directory browsing HTML once authenticated. This page is always +gzipped. If the `Accept: application/json` header is provided, then the JSON representation of the +root will be returned. + +#### OPTIONS +When requested with the `OPTIONS` method, the server will respond with . + +#### `/fs//` +Directory paths must end with a /. Otherwise, the path is assumed to be a file. + +##### GET +Returns a JSON representation of the directory. + +* `200 OK` - Directory exists and JSON returned +* `404 Not Found` - Missing directory + +##### PUT +Tries to make a directory at the given path. Request body is ignored. Returns: + +* `204 No Content` - Directory exists +* `201 Created` - Directory created +* `409 Conflict` - USB is active and preventing file system modification +* `404 Not Found` - Missing parent directory +* `500 Server Error` - Other, unhandled error + +Example: + +``sh +curl -v -u :passw0rd -X PUT -L --location-trusted http://circuitpython.local/fs/lib/hello/world/ +`` + +##### DELETE +Deletes the directory and all of its contents. + + +* `404 Not Found` - No directory +* `409 Conflict` - USB is active and preventing file system modification + +Example: + +``sh +curl -v -u :passw0rd -X DELETE -L --location-trusted http://circuitpython.local/fs/lib/hello/world/ +`` + + +#### `/fs/` + +##### GET +Returns the raw file contents. `Content-Type` will be set based on extension: + +* `text/plain` - `.py`, `.txt` +* `text/javascript` - `.js` +* `text/html` - `.html` +* `application/json` - `.json` +* `application/octet-stream` - Everything else + +Will return: +* `200 OK` - File exists and file returned +* `404 Not Found` - Missing file + +##### PUT +Stores the provided content to the file path. Returns: + +* `201 Created` - File created and saved +* `204 No Content` - File existed and overwritten +* `404 Not Found` - Missing parent directory +* `409 Conflict` - USB is active and preventing file system modification +* `413 Payload Too Large` - `Expect` header not sent and file is too large +* `417 Expectation Failed` - `Expect` header sent and file is too large +* `500 Server Error` - Other, unhandled error + +If the client sends the `Expect` header, the server will reply with `100 Continue` when ok. + +##### DELETE +Deletes the file. + +* `404 Not Found` - File not found +* `409 Conflict` - USB is active and preventing file system modification + +Example: + +``sh +curl -v -u :passw0rd -X DELETE -L --location-trusted http://circuitpython.local/fs/lib/hello/world.txt +`` + +### `/cp/` + +`/cp/` serves basic info about the CircuitPython device and others discovered through MDNS. It is +not protected by basic auth in case the device is someone elses. + +Only `GET` requests are supported and will return `XXX Method Not Allowed` otherwise. + +#### `/cp/version.json` + +Returns information about the device. + +* `web_api_version`: Always `1`. This versions the rest of the API and new versions may not be backwards compatible. +* `version`: CircuitPython build version. +* `build_date`: CircuitPython build date. +* `board_name`: Human readable name of the board. +* `mcu_name`: Human readable name of the microcontroller. +* `board_id`: Board id used in code and on circuitpython.org. +* `creator_id`: Creator ID for the board. +* `creation_id`: Creation ID for the board, set by the creator. +* `hostname`: MDNS hostname. +* `port`: Port of CircuitPython Web Service. +* `ip`: IP address of the device. + +Example: +```sh +curl -v -L http://circuitpython.local/cp/version.json +``` + +```json +{ + "web_api_version": 1, + "version": "8.0.0-alpha.1-20-ge1d4518a9-dirty", + "build_date": "2022-06-24", + "board_name": "ESP32-S3-USB-OTG-N8", + "mcu_name": "ESP32S3", + "board_id": "espressif_esp32s3_usb_otg_n8", + "creator_id": 12346, + "creation_id": 28683, + "hostname": "cpy-f57ce8", + "port": 80, + "ip": "192.168.1.94" +} +``` + +#### `/cp/devices.json` + +Returns information about other devices found on the network using MDNS. + +* `total`: Total MDNS response count. May be more than in `devices` if internal limits were hit. +* `devices`: List of discovered devices. + * `hostname`: MDNS hostname + * `instance_name`: MDNS instance name. Defaults to human readable board name. + * `port`: Port of CircuitPython Web API + * `ip`: IP address + +Example: +```sh +curl -v -L http://circuitpython.local/cp/devices.json +``` + +```json +{ + "total": 1, + "devices": [ + { + "hostname": "cpy-951032", + "instance_name": "Adafruit Feather ESP32-S2 TFT", + "port": 80, + "ip": "192.168.1.235" + } + ] +} +``` + +### Static files + +* `/favicon.ico` - Blinka +* `/directory.js` - JavaScript for `/fs/` +* `/welcome.js` - JavaScript for `/` + +### WebSocket + +Coming soon! diff --git a/ports/espressif/common-hal/mdns/RemoteService.c b/ports/espressif/common-hal/mdns/RemoteService.c index a19fe8cc20..5a84c4a79b 100644 --- a/ports/espressif/common-hal/mdns/RemoteService.c +++ b/ports/espressif/common-hal/mdns/RemoteService.c @@ -63,7 +63,7 @@ mp_int_t common_hal_mdns_remoteservice_get_port(mdns_remoteservice_obj_t *self) return self->result->port; } -uint32_t mdns_remoteservice_get_ipv4(mdns_remoteservice_obj_t *self) { +uint32_t mdns_remoteservice_get_ipv4_address(mdns_remoteservice_obj_t *self) { if (self->result == NULL || self->result->ip_protocol != MDNS_IP_PROTOCOL_V4 || self->result->addr == NULL) { @@ -81,8 +81,8 @@ uint32_t mdns_remoteservice_get_ipv4(mdns_remoteservice_obj_t *self) { return 0; } -mp_obj_t common_hal_mdns_remoteservice_get_ipv4(mdns_remoteservice_obj_t *self) { - uint32_t addr = mdns_remoteservice_get_ipv4(self); +mp_obj_t common_hal_mdns_remoteservice_get_ipv4_address(mdns_remoteservice_obj_t *self) { + uint32_t addr = mdns_remoteservice_get_ipv4_address(self); if (addr == 0) { return mp_const_none; } diff --git a/ports/espressif/common-hal/mdns/Server.c b/ports/espressif/common-hal/mdns/Server.c index 6cc562a34c..3c0070ae6f 100644 --- a/ports/espressif/common-hal/mdns/Server.c +++ b/ports/espressif/common-hal/mdns/Server.c @@ -49,7 +49,6 @@ void mdns_server_construct(mdns_server_obj_t *self, bool workflow) { if (workflow) { // Set a delegated entry to ourselves. This allows us to respond to "circuitpython.local" // queries as well. - // TODO: Allow for disabling this with `supervisor.disable_web_workflow()`. mdns_ip_addr_t our_ip; esp_netif_get_ip_info(common_hal_wifi_radio_obj.netif, &common_hal_wifi_radio_obj.ip_info); our_ip.next = NULL; diff --git a/ports/espressif/common-hal/socketpool/Socket.c b/ports/espressif/common-hal/socketpool/Socket.c index fcfc95873c..0774204bad 100644 --- a/ports/espressif/common-hal/socketpool/Socket.c +++ b/ports/espressif/common-hal/socketpool/Socket.c @@ -37,14 +37,9 @@ #include "components/lwip/lwip/src/include/lwip/sys.h" #include "components/lwip/lwip/src/include/lwip/netdb.h" -#include "esp_log.h" - -static const char *TAG = "socket"; - STATIC socketpool_socket_obj_t *open_socket_handles[CONFIG_LWIP_MAX_SOCKETS]; void socket_user_reset(void) { - ESP_LOGW(TAG, "Reset sockets"); for (size_t i = 0; i < MP_ARRAY_SIZE(open_socket_handles); i++) { if (open_socket_handles[i]) { if (open_socket_handles[i]->num > 0) { @@ -83,9 +78,6 @@ int socketpool_socket_accept(socketpool_socket_obj_t *self, uint8_t *ip, uint32_ newsoc = lwip_accept(self->num, (struct sockaddr *)&accept_addr, &socklen); // In non-blocking mode, fail instead of timing out if (newsoc == -1 && (self->timeout_ms == 0 || mp_hal_is_interrupted())) { - if (errno != EAGAIN) { - ESP_LOGE(TAG, "accept failed %d", errno); - } return -MP_EAGAIN; } } @@ -227,9 +219,7 @@ bool common_hal_socketpool_socket_get_connected(socketpool_socket_obj_t *self) { } bool common_hal_socketpool_socket_listen(socketpool_socket_obj_t *self, int backlog) { - int result = lwip_listen(self->num, backlog); - ESP_LOGE(TAG, "listen result %d", result); - return result == 0; + return lwip_listen(self->num, backlog); } mp_uint_t common_hal_socketpool_socket_recvfrom_into(socketpool_socket_obj_t *self, @@ -294,8 +284,6 @@ int socketpool_socket_recv_into(socketpool_socket_obj_t *self, if (errno == ENOTCONN) { self->connected = false; return -MP_ENOTCONN; - } else if (errno != EAGAIN) { - ESP_LOGE(TAG, "recv %d", errno); } return -MP_EAGAIN; } diff --git a/ports/espressif/common-hal/wifi/Radio.c b/ports/espressif/common-hal/wifi/Radio.c index fae1f77514..94345fdf90 100644 --- a/ports/espressif/common-hal/wifi/Radio.c +++ b/ports/espressif/common-hal/wifi/Radio.c @@ -48,10 +48,6 @@ #define MAC_ADDRESS_LENGTH 6 -#include "esp_log.h" - -static const char *TAG = "radio"; - static void set_mode_station(wifi_radio_obj_t *self, bool state) { wifi_mode_t next_mode; if (state) { @@ -240,12 +236,11 @@ wifi_radio_error_t common_hal_wifi_radio_connect(wifi_radio_obj_t *self, uint8_t if (!common_hal_wifi_radio_get_enabled(self)) { mp_raise_RuntimeError(translate("wifi is not enabled")); } - ESP_LOGI(TAG, "connect"); wifi_config_t *config = &self->sta_config; - // size_t timeout_ms = timeout * 1000; - // uint32_t start_time = common_hal_time_monotonic_ms(); - // uint32_t end_time = start_time + timeout_ms; + size_t timeout_ms = timeout * 1000; + uint32_t start_time = common_hal_time_monotonic_ms(); + uint32_t end_time = start_time + timeout_ms; EventBits_t bits; // can't block since both bits are false after wifi_init @@ -309,9 +304,7 @@ wifi_radio_error_t common_hal_wifi_radio_connect(wifi_radio_obj_t *self, uint8_t esp_wifi_set_config(ESP_IF_WIFI_STA, config); self->starting_retries = 5; self->retries_left = 5; - ESP_LOGI(TAG, "wifi connect"); esp_wifi_connect(); - ESP_LOGI(TAG, "wifi connect done"); do { RUN_BACKGROUND_TASKS; @@ -321,11 +314,11 @@ wifi_radio_error_t common_hal_wifi_radio_connect(wifi_radio_obj_t *self, uint8_t pdTRUE, 0); // Don't retry anymore if we're over our time budget. - // if (self->retries_left > 0 && common_hal_time_monotonic_ms() > end_time) { - // self->retries_left = 0; - // } + if (self->retries_left > 0 && common_hal_time_monotonic_ms() > end_time) { + self->retries_left = 0; + } } while ((bits & (WIFI_CONNECTED_BIT | WIFI_DISCONNECTED_BIT)) == 0 && !mp_hal_is_interrupted()); - ESP_LOGI(TAG, "connect done"); + if ((bits & WIFI_DISCONNECTED_BIT) != 0) { if (self->last_disconnect_reason == WIFI_REASON_AUTH_FAIL) { return WIFI_RADIO_ERROR_AUTH_FAIL; diff --git a/ports/espressif/common-hal/wifi/__init__.c b/ports/espressif/common-hal/wifi/__init__.c index a827e3ba64..61af5b774f 100644 --- a/ports/espressif/common-hal/wifi/__init__.c +++ b/ports/espressif/common-hal/wifi/__init__.c @@ -168,7 +168,6 @@ void common_hal_wifi_init(bool user_initiated) { void wifi_user_reset(void) { if (wifi_user_initiated) { - ESP_LOGW(TAG, "Reset wifi"); wifi_reset(); } } diff --git a/ports/espressif/esp-idf-config/sdkconfig.defaults b/ports/espressif/esp-idf-config/sdkconfig.defaults index 121c20ad34..f30e8c1109 100644 --- a/ports/espressif/esp-idf-config/sdkconfig.defaults +++ b/ports/espressif/esp-idf-config/sdkconfig.defaults @@ -297,7 +297,7 @@ CONFIG_ESP_SYSTEM_EVENT_TASK_STACK_SIZE=2304 CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192 CONFIG_ESP_MINIMAL_SHARED_STACK_SIZE=2048 CONFIG_ESP_INT_WDT=y -CONFIG_ESP_INT_WDT_TIMEOUT_MS=1000 +CONFIG_ESP_INT_WDT_TIMEOUT_MS=300 CONFIG_ESP_INT_WDT_CHECK_CPU1=y # CONFIG_ESP_TASK_WDT is not set # CONFIG_ESP_PANIC_HANDLER_IRAM is not set diff --git a/shared-bindings/mdns/RemoteService.c b/shared-bindings/mdns/RemoteService.c index 846207b69b..49eda58557 100644 --- a/shared-bindings/mdns/RemoteService.c +++ b/shared-bindings/mdns/RemoteService.c @@ -106,6 +106,18 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_1(mdns_remoteservice_get_port_obj, remoteservice_ MP_PROPERTY_GETTER(mdns_remoteservice_port_obj, (mp_obj_t)&mdns_remoteservice_get_port_obj); +//| ipv4_address: Optional[ipaddress.IPv4Address] +//| """IP v4 Address of the remote service. None if no A records are found.""" +//| +STATIC mp_obj_t _mdns_remoteservice_get_ipv4_address(mp_obj_t self) { + return common_hal_mdns_remoteservice_get_ipv4_address(self); + +} +MP_DEFINE_CONST_FUN_OBJ_1(mdns_remoteservice_get_ipv4_address_obj, _mdns_remoteservice_get_ipv4_address); + +MP_PROPERTY_GETTER(mdns_remoteservice_ipv4_address_obj, + (mp_obj_t)&mdns_remoteservice_get_ipv4_address_obj); + //| def __del__(self) -> None: //| """Deletes the RemoteService object.""" //| ... @@ -123,6 +135,7 @@ STATIC const mp_rom_map_elem_t mdns_remoteservice_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_service_type), MP_ROM_PTR(&mdns_remoteservice_service_type_obj) }, { MP_ROM_QSTR(MP_QSTR_protocol), MP_ROM_PTR(&mdns_remoteservice_protocol_obj) }, { MP_ROM_QSTR(MP_QSTR_port), MP_ROM_PTR(&mdns_remoteservice_port_obj) }, + { MP_ROM_QSTR(MP_QSTR_ipv4_address), MP_ROM_PTR(&mdns_remoteservice_ipv4_address_obj) }, { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&mdns_remoteservice_deinit_obj) }, }; diff --git a/shared-bindings/mdns/RemoteService.h b/shared-bindings/mdns/RemoteService.h index 5a9d3d5e5b..4783170e88 100644 --- a/shared-bindings/mdns/RemoteService.h +++ b/shared-bindings/mdns/RemoteService.h @@ -38,8 +38,8 @@ const char *common_hal_mdns_remoteservice_get_protocol(mdns_remoteservice_obj_t const char *common_hal_mdns_remoteservice_get_instance_name(mdns_remoteservice_obj_t *self); const char *common_hal_mdns_remoteservice_get_hostname(mdns_remoteservice_obj_t *self); mp_int_t common_hal_mdns_remoteservice_get_port(mdns_remoteservice_obj_t *self); -mp_obj_t common_hal_mdns_remoteservice_get_ipv4(mdns_remoteservice_obj_t *self); +mp_obj_t common_hal_mdns_remoteservice_get_ipv4_address(mdns_remoteservice_obj_t *self); void common_hal_mdns_remoteservice_deinit(mdns_remoteservice_obj_t *self); // For internal use. -uint32_t mdns_remoteservice_get_ipv4(mdns_remoteservice_obj_t *self); +uint32_t mdns_remoteservice_get_ipv4_address(mdns_remoteservice_obj_t *self); diff --git a/supervisor/shared/web_workflow/web_workflow.c b/supervisor/shared/web_workflow/web_workflow.c index d5c1cf5a82..1e1196ce4f 100644 --- a/supervisor/shared/web_workflow/web_workflow.c +++ b/supervisor/shared/web_workflow/web_workflow.c @@ -608,7 +608,7 @@ static void _reply_with_devices_json(socketpool_socket_obj_t *socket, _request * _send_chunk(socket, ", \"ip\": \""); char ip_encoded[4 * 4]; - uint32_t ipv4_address = mdns_remoteservice_get_ipv4(&found_devices[i]); + uint32_t ipv4_address = mdns_remoteservice_get_ipv4_address(&found_devices[i]); uint8_t *octets = (uint8_t *)&ipv4_address; snprintf(ip_encoded, sizeof(ip_encoded), "%d.%d.%d.%d", octets[0], octets[1], octets[2], octets[3]); _send_chunk(socket, ip_encoded);