From 7589e53fea04d2bada44b7729432a3ee2a9237a4 Mon Sep 17 00:00:00 2001 From: Scott Shawcroft Date: Wed, 29 Jun 2022 16:31:55 -0700 Subject: [PATCH 01/10] WIP websocket accept and hashlib --- locale/circuitpython.pot | 4 + ports/espressif/common-hal/hashlib/Hash.c | 52 +++++++++ ports/espressif/common-hal/hashlib/Hash.h | 41 +++++++ ports/espressif/common-hal/hashlib/__init__.c | 40 +++++++ ports/espressif/mpconfigport.mk | 1 + py/circuitpy_defns.mk | 5 + shared-bindings/hashlib/Hash.c | 103 ++++++++++++++++++ shared-bindings/hashlib/Hash.h | 43 ++++++++ shared-bindings/hashlib/__init__.c | 90 +++++++++++++++ shared-bindings/hashlib/__init__.h | 36 ++++++ .../shared/web_workflow/static/serial.html | 16 +++ .../shared/web_workflow/static/serial.js | 43 ++++++++ .../shared/web_workflow/static/welcome.html | 2 +- supervisor/shared/web_workflow/web_workflow.c | 40 +++++++ 14 files changed, 515 insertions(+), 1 deletion(-) create mode 100644 ports/espressif/common-hal/hashlib/Hash.c create mode 100644 ports/espressif/common-hal/hashlib/Hash.h create mode 100644 ports/espressif/common-hal/hashlib/__init__.c create mode 100644 shared-bindings/hashlib/Hash.c create mode 100644 shared-bindings/hashlib/Hash.h create mode 100644 shared-bindings/hashlib/__init__.c create mode 100644 shared-bindings/hashlib/__init__.h create mode 100644 supervisor/shared/web_workflow/static/serial.html create mode 100644 supervisor/shared/web_workflow/static/serial.js diff --git a/locale/circuitpython.pot b/locale/circuitpython.pot index 5740e1e288..a66a386f1d 100644 --- a/locale/circuitpython.pot +++ b/locale/circuitpython.pot @@ -2199,6 +2199,10 @@ msgstr "" msgid "Unsupported format" msgstr "" +#: shared-bindings/hashlib/__init__.c +msgid "Unsupported hash algorithm" +msgstr "" + #: ports/espressif/common-hal/dualbank/__init__.c msgid "Update Failed" msgstr "" diff --git a/ports/espressif/common-hal/hashlib/Hash.c b/ports/espressif/common-hal/hashlib/Hash.c new file mode 100644 index 0000000000..8e2ab02edb --- /dev/null +++ b/ports/espressif/common-hal/hashlib/Hash.c @@ -0,0 +1,52 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2022 Scott Shawcroft for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "shared-bindings/hashlib/Hash.h" + +#include "components/mbedtls/mbedtls/include/mbedtls/ssl.h" + +void common_hal_hashlib_hash_update(hashlib_hash_obj_t *self, const uint8_t *data, size_t datalen) { + if (self->hash_type == MBEDTLS_SSL_HASH_SHA1) { + mbedtls_sha1_update_ret(&self->sha1, data, datalen); + return; + } +} + +void common_hal_hashlib_hash_digest(hashlib_hash_obj_t *self, uint8_t *data, size_t datalen) { + if (datalen < common_hal_hashlib_hash_get_digest_size(self)) { + return; + } + if (self->hash_type == MBEDTLS_SSL_HASH_SHA1) { + mbedtls_sha1_finish_ret(&self->sha1, data); + } +} + +size_t common_hal_hashlib_hash_get_digest_size(hashlib_hash_obj_t *self) { + if (self->hash_type == MBEDTLS_SSL_HASH_SHA1) { + return 20; + } + return 0; +} diff --git a/ports/espressif/common-hal/hashlib/Hash.h b/ports/espressif/common-hal/hashlib/Hash.h new file mode 100644 index 0000000000..ece282833e --- /dev/null +++ b/ports/espressif/common-hal/hashlib/Hash.h @@ -0,0 +1,41 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2022 Scott Shawcroft for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef MICROPY_INCLUDED_ESPRESSIF_COMMON_HAL_HASHLIB_HASH_H +#define MICROPY_INCLUDED_ESPRESSIF_COMMON_HAL_HASHLIB_HASH_H + +#include "components/mbedtls/mbedtls/include/mbedtls/sha1.h" + +typedef struct { + mp_obj_base_t base; + union { + mbedtls_sha1_context sha1; + }; + // Of MBEDTLS_SSL_HASH_* + uint8_t hash_type; +} hashlib_hash_obj_t; + +#endif // MICROPY_INCLUDED_ESPRESSIF_COMMON_HAL_HASHLIB_HASH_H diff --git a/ports/espressif/common-hal/hashlib/__init__.c b/ports/espressif/common-hal/hashlib/__init__.c new file mode 100644 index 0000000000..1e6b2a4802 --- /dev/null +++ b/ports/espressif/common-hal/hashlib/__init__.c @@ -0,0 +1,40 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2022 Scott Shawcroft for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "shared-bindings/hashlib/__init__.h" + +#include "components/mbedtls/mbedtls/include/mbedtls/ssl.h" + + +bool common_hal_hashlib_new(hashlib_hash_obj_t *self, const char *algorithm) { + if (strcmp(algorithm, "sha1") == 0) { + self->hash_type = MBEDTLS_SSL_HASH_SHA1; + mbedtls_sha1_init(&self->sha1); + mbedtls_sha1_starts_ret(&self->sha1); + return true; + } + return false; +} diff --git a/ports/espressif/mpconfigport.mk b/ports/espressif/mpconfigport.mk index 2eeb86e49d..ddaa4836d3 100644 --- a/ports/espressif/mpconfigport.mk +++ b/ports/espressif/mpconfigport.mk @@ -19,6 +19,7 @@ CIRCUITPY_COUNTIO ?= 1 CIRCUITPY_DUALBANK ?= 1 CIRCUITPY_FRAMEBUFFERIO ?= 1 CIRCUITPY_FREQUENCYIO ?= 1 +CIRCUITPY_HASHLIB ?= 1 CIRCUITPY_IMAGECAPTURE ?= 1 CIRCUITPY_I2CPERIPHERAL ?= 1 CIRCUITPY_RGBMATRIX ?= 1 diff --git a/py/circuitpy_defns.mk b/py/circuitpy_defns.mk index c5c89ab19f..af49c876ba 100644 --- a/py/circuitpy_defns.mk +++ b/py/circuitpy_defns.mk @@ -207,6 +207,9 @@ endif ifeq ($(CIRCUITPY_GNSS),1) SRC_PATTERNS += gnss/% endif +ifeq ($(CIRCUITPY_HASHLIB),1) +SRC_PATTERNS += hashlib/% +endif ifeq ($(CIRCUITPY_I2CPERIPHERAL),1) SRC_PATTERNS += i2cperipheral/% endif @@ -419,6 +422,8 @@ SRC_COMMON_HAL_ALL = \ gnss/GNSS.c \ gnss/PositionFix.c \ gnss/SatelliteSystem.c \ + hashlib/__init__.c \ + hashlib/Hash.c \ i2cperipheral/I2CPeripheral.c \ i2cperipheral/__init__.c \ microcontroller/Pin.c \ diff --git a/shared-bindings/hashlib/Hash.c b/shared-bindings/hashlib/Hash.c new file mode 100644 index 0000000000..5dab05fa10 --- /dev/null +++ b/shared-bindings/hashlib/Hash.c @@ -0,0 +1,103 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2022 Scott Shawcroft for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "shared-bindings/hashlib/Hash.h" + +// #include "shared-bindings/util.h" + +// #include "shared/runtime/buffer_helper.h" +// #include "shared/runtime/interrupt_char.h" + +// #include "py/mperrno.h" +// #include "py/mphal.h" +#include "py/obj.h" +#include "py/objproperty.h" +#include "py/objstr.h" +#include "py/runtime.h" + +//| class Hash: +//| """In progress hash algorithm. This object is always created by a `hashlib.new()`. It has no +//| user-visible constructor.""" +//| + +//| digest_size: int +//| """Digest size in bytes""" +//| +STATIC mp_obj_t hashlib_hash_get_digest_size(mp_obj_t self_in) { + mp_check_self(mp_obj_is_type(self_in, &hashlib_hash_type)); + hashlib_hash_obj_t *self = MP_OBJ_TO_PTR(self_in); + return MP_OBJ_NEW_SMALL_INT(common_hal_hashlib_hash_get_digest_size(self)); +} +MP_PROPERTY_GETTER(hashlib_hash_digest_size_obj, hashlib_hash_get_digest_size); + +//| def update(self, data: ReadableBuffer) -> None: +//| """Update the hash with the given bytes. +//| +//| :param ~circuitpython_typing.ReadableBuffer data: Update the hash from data in this buffer""" +//| ... +//| +mp_obj_t hashlib_hash_update(mp_obj_t self_in, mp_obj_t buf_in) { + mp_check_self(mp_obj_is_type(self_in, &hashlib_hash_type)); + hashlib_hash_obj_t *self = MP_OBJ_TO_PTR(self_in); + + mp_buffer_info_t bufinfo; + mp_get_buffer_raise(buf_in, &bufinfo, MP_BUFFER_READ); + + common_hal_hashlib_hash_update(self, bufinfo.buf, bufinfo.len); + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_2(hashlib_hash_update_obj, hashlib_hash_update); + +//| def digest(self) -> bytes: +//| """Returns the current digest as bytes() with a length of `hashlib.Hash.digest_size`.""" +//| ... +//| +STATIC mp_obj_t hashlib_hash_digest(mp_obj_t self_in) { + mp_check_self(mp_obj_is_type(self_in, &hashlib_hash_type)); + hashlib_hash_obj_t *self = MP_OBJ_TO_PTR(self_in); + + size_t size = common_hal_hashlib_hash_get_digest_size(self); + mp_obj_t obj = mp_obj_new_bytes_of_zeros(size); + mp_obj_str_t *o = MP_OBJ_TO_PTR(obj); + + common_hal_hashlib_hash_digest(self, (uint8_t *)o->data, size); + return obj; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(hashlib_hash_digest_obj, hashlib_hash_digest); + +STATIC const mp_rom_map_elem_t hashlib_hash_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_digest_size), MP_ROM_PTR(&hashlib_hash_digest_size_obj) }, + { MP_ROM_QSTR(MP_QSTR_update), MP_ROM_PTR(&hashlib_hash_update_obj) }, + { MP_ROM_QSTR(MP_QSTR_digest), MP_ROM_PTR(&hashlib_hash_digest_obj) }, +}; + +STATIC MP_DEFINE_CONST_DICT(hashlib_hash_locals_dict, hashlib_hash_locals_dict_table); + +const mp_obj_type_t hashlib_hash_type = { + { &mp_type_type }, + .name = MP_QSTR_Hash, + .locals_dict = (mp_obj_dict_t *)&hashlib_hash_locals_dict, +}; diff --git a/shared-bindings/hashlib/Hash.h b/shared-bindings/hashlib/Hash.h new file mode 100644 index 0000000000..f3845b02ff --- /dev/null +++ b/shared-bindings/hashlib/Hash.h @@ -0,0 +1,43 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2022 Scott Shawcroft for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef MICROPY_INCLUDED_SHARED_BINDINGS_HASHLIB_HASH_H +#define MICROPY_INCLUDED_SHARED_BINDINGS_HASHLIB_HASH_H + +#include "py/obj.h" + +#include "common-hal/hashlib/Hash.h" + +extern const mp_obj_type_t hashlib_hash_type; + +// So that new can call it when given data. +mp_obj_t hashlib_hash_update(mp_obj_t self_in, mp_obj_t buf_in); + +void common_hal_hashlib_hash_update(hashlib_hash_obj_t *self, const uint8_t *data, size_t datalen); +void common_hal_hashlib_hash_digest(hashlib_hash_obj_t *self, uint8_t *data, size_t datalen); +size_t common_hal_hashlib_hash_get_digest_size(hashlib_hash_obj_t *self); + +#endif // MICROPY_INCLUDED_SHARED_BINDINGS_HASHLIB_HASH_H diff --git a/shared-bindings/hashlib/__init__.c b/shared-bindings/hashlib/__init__.c new file mode 100644 index 0000000000..4b5be0165b --- /dev/null +++ b/shared-bindings/hashlib/__init__.c @@ -0,0 +1,90 @@ +/* + * This file is part of the Micro Python project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2022 Scott Shawcroft for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include + +#include "py/obj.h" +#include "py/mpconfig.h" +#include "py/runtime.h" +#include "shared-bindings/hashlib/__init__.h" +#include "shared-bindings/hashlib/Hash.h" +#include "supervisor/shared/translate/translate.h" + +//| """Hashing related functions +//| +//| |see_cpython_module| :mod:`cpython:hashlib`. +//| """ +//| +//| def new(name, data=b"") -> hashlib.Hash: +//| """Returns a Hash object setup for the named algorithm. Raises ValueError when the named +//| algorithm is unsupported. +//| +//| :return: a hash object for the given algorithm +//| :rtype: hashlib.Hash""" +//| ... +//| +STATIC mp_obj_t hashlib_new(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { + enum { ARG_name, ARG_data }; + static const mp_arg_t allowed_args[] = { + { MP_QSTR_name, MP_ARG_REQUIRED | MP_ARG_OBJ }, + { MP_QSTR_data, MP_ARG_OBJ, {.u_obj = mp_const_none} }, + }; + mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; + mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args); + + const char *algorithm = mp_obj_str_get_str(args[ARG_name].u_obj); + + hashlib_hash_obj_t *self = m_new_obj(hashlib_hash_obj_t); + self->base.type = &hashlib_hash_type; + + if (!common_hal_hashlib_new(self, algorithm)) { + mp_raise_ValueError(translate("Unsupported hash algorithm")); + } + + if (args[ARG_data].u_obj != mp_const_none) { + hashlib_hash_update(self, args[ARG_data].u_obj); + } + return self; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_KW(hashlib_new_obj, 1, hashlib_new); + +STATIC const mp_rom_map_elem_t hashlib_module_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_hashlib) }, + + { MP_ROM_QSTR(MP_QSTR_new), MP_ROM_PTR(&hashlib_new_obj) }, + + // Hash is deliberately omitted here because CPython doesn't expose the + // object on `hashlib` only the internal `_hashlib`. +}; + +STATIC MP_DEFINE_CONST_DICT(hashlib_module_globals, hashlib_module_globals_table); + +const mp_obj_module_t hashlib_module = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&hashlib_module_globals, +}; + +MP_REGISTER_MODULE(MP_QSTR_hashlib, hashlib_module, CIRCUITPY_HASHLIB); diff --git a/shared-bindings/hashlib/__init__.h b/shared-bindings/hashlib/__init__.h new file mode 100644 index 0000000000..ae47b546a4 --- /dev/null +++ b/shared-bindings/hashlib/__init__.h @@ -0,0 +1,36 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2022 Scott Shawcroft for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef MICROPY_INCLUDED_SHARED_BINDINGS_HASHLIB___INIT___H +#define MICROPY_INCLUDED_SHARED_BINDINGS_HASHLIB___INIT___H + +#include + +#include "shared-bindings/hashlib/Hash.h" + +bool common_hal_hashlib_new(hashlib_hash_obj_t *self, const char *algorithm); + +#endif // MICROPY_INCLUDED_SHARED_BINDINGS_HASHLIB___INIT___H diff --git a/supervisor/shared/web_workflow/static/serial.html b/supervisor/shared/web_workflow/static/serial.html new file mode 100644 index 0000000000..766a7a23e2 --- /dev/null +++ b/supervisor/shared/web_workflow/static/serial.html @@ -0,0 +1,16 @@ + + + + Simple client + + + + + 

+  

+    
+    
+    
+  

+
+
diff --git a/supervisor/shared/web_workflow/static/serial.js b/supervisor/shared/web_workflow/static/serial.js
new file mode 100644
index 0000000000..36c980dc08
--- /dev/null
+++ b/supervisor/shared/web_workflow/static/serial.js
@@ -0,0 +1,43 @@
+
+var ws;
+
+function onSubmit() {
+  var input = document.getElementById("input");
+  // You can send message to the Web Socket using ws.send.
+  ws.send(input.value);
+  output("send: " + input.value);
+  input.value = "";
+  input.focus();
+}
+
+function onCloseClick() {
+  ws.close();
+}
+
+function output(str) {
+  var log = document.getElementById("log");
+  log.innerHTML += str;
+}
+
+// Connect to Web Socket
+ws = new WebSocket("ws://cpy-f57ce8.local/cp/serial/");
+// ws = new WebSocket("ws://127.0.0.1:9001")
+
+// Set event handlers.
+ws.onopen = function() {
+  output("onopen");
+};
+
+ws.onmessage = function(e) {
+  // e.data contains received string.
+  output("onmessage: " + e.data);
+};
+
+ws.onclose = function() {
+  output("onclose");
+};
+
+ws.onerror = function(e) {
+  output("onerror");
+  console.log(e)
+};
diff --git a/supervisor/shared/web_workflow/static/welcome.html b/supervisor/shared/web_workflow/static/welcome.html
index 7fb68c302e..6ef1ccd767 100644
--- a/supervisor/shared/web_workflow/static/welcome.html
+++ b/supervisor/shared/web_workflow/static/welcome.html
@@ -3,8 +3,8 @@
 
     CircuitPython
     
+    
 
-
 
     
 Welcome!

     Welcome to CircuitPython's Web API. Go to the file browser to work with files in the CIRCUITPY drive. Make sure you've set CIRCUITPY_WEB_API_PASSWORD='somepassword' in /.env. Provide the password when the browser prompts for it. Leave the username blank.
diff --git a/supervisor/shared/web_workflow/web_workflow.c b/supervisor/shared/web_workflow/web_workflow.c
index cbe1b24e86..b9a1de67f2 100644
--- a/supervisor/shared/web_workflow/web_workflow.c
+++ b/supervisor/shared/web_workflow/web_workflow.c
@@ -86,6 +86,10 @@ typedef struct {
     bool authenticated;
     bool expect;
     bool json;
+    bool websocket;
+    uint32_t websocket_version;
+    // RFC6455 for websockets says this header should be 24 base64 characters long.
+    char websocket_key[24 + 1];
 } _request;
 
 static wifi_radio_error_t wifi_status = WIFI_RADIO_ERROR_NONE;
@@ -519,6 +523,7 @@ static void _reply_redirect(socketpool_socket_obj_t *socket, _request *request,
         "Location: http://", hostname, ".local", path, "\r\n", NULL);
     _cors_header(socket, request);
     _send_str(socket, "\r\n");
+    ESP_LOGI(TAG, "redirect");
 }
 
 static void _reply_directory_json(socketpool_socket_obj_t *socket, _request *request, FF_DIR *dir, const char *request_path, const char *path) {
@@ -826,6 +831,8 @@ STATIC_FILE(directory_html);
 STATIC_FILE(directory_js);
 STATIC_FILE(welcome_html);
 STATIC_FILE(welcome_js);
+STATIC_FILE(serial_html);
+STATIC_FILE(serial_js);
 STATIC_FILE(blinka_16x16_ico);
 
 static void _reply_static(socketpool_socket_obj_t *socket, _request *request, const uint8_t *response, size_t response_len, const char *content_type) {
@@ -844,6 +851,13 @@ static void _reply_static(socketpool_socket_obj_t *socket, _request *request, co
 
 #define _REPLY_STATIC(socket, request, filename) _reply_static(socket, request, filename, filename##_length, filename##_content_type)
 
+static void _reply_websocket_upgrade(socketpool_socket_obj_t *socket, _request *request) {
+    ESP_LOGI(TAG, "websocket!");
+    // Compute accept key
+    // Reply with upgrade
+    // Copy socket state into websocket and mark given socket as closed even though it isn't actually.
+}
+
 static bool _reply(socketpool_socket_obj_t *socket, _request *request) {
     if (request->redirect) {
         _reply_redirect(socket, request, request->path);
@@ -980,6 +994,22 @@ static bool _reply(socketpool_socket_obj_t *socket, _request *request) {
             _reply_with_devices_json(socket, request);
         } else if (strcmp(path, "/version.json") == 0) {
             _reply_with_version_json(socket, request);
+        } else if (strcmp(path, "/serial/") == 0) {
+            if (!request->authenticated) {
+                if (_api_password[0] != '\0') {
+                    _reply_unauthorized(socket, request);
+                } else {
+                    _reply_forbidden(socket, request);
+                }
+            } else if (request->websocket) {
+                ESP_LOGI(TAG, "websocket!");
+                _reply_websocket_upgrade(socket, request);
+            } else {
+                _REPLY_STATIC(socket, request, serial_html);
+            }
+            _reply_with_version_json(socket, request);
+        } else {
+            _reply_missing(socket, request);
         }
     } else if (strcmp(request->method, "GET") != 0) {
         _reply_method_not_allowed(socket, request);
@@ -992,6 +1022,8 @@ static bool _reply(socketpool_socket_obj_t *socket, _request *request) {
             _REPLY_STATIC(socket, request, directory_js);
         } else if (strcmp(request->path, "/welcome.js") == 0) {
             _REPLY_STATIC(socket, request, welcome_js);
+        } else if (strcmp(request->path, "/serial.js") == 0) {
+            _REPLY_STATIC(socket, request, serial_js);
         } else if (strcmp(request->path, "/favicon.ico") == 0) {
             // TODO: Autogenerate this based on the blinka bitmap and change the
             // palette based on MAC address.
@@ -1015,6 +1047,7 @@ static void _reset_request(_request *request) {
     request->authenticated = false;
     request->expect = false;
     request->json = false;
+    request->websocket = false;
 }
 
 static void _process_request(socketpool_socket_obj_t *socket, _request *request) {
@@ -1111,6 +1144,13 @@ static void _process_request(socketpool_socket_obj_t *socket, _request *request)
                         strcpy(request->origin, request->header_value);
                     } else if (strcmp(request->header_key, "X-Timestamp") == 0) {
                         request->timestamp_ms = strtoull(request->header_value, NULL, 10);
+                    } else if (strcmp(request->header_key, "Upgrade") == 0) {
+                        request->websocket = strcmp(request->header_value, "websocket") == 0;
+                    } else if (strcmp(request->header_key, "Sec-WebSocket-Version") == 0) {
+                        request->websocket_version = strtoul(request->header_value, NULL, 10);
+                    } else if (strcmp(request->header_key, "Sec-WebSocket-Key") == 0 &&
+                               strlen(request->header_value) == 24) {
+                        strcpy(request->websocket_key, request->header_value);
                     }
                     ESP_LOGI(TAG, "Header %s %s", request->header_key, request->header_value);
                 } else if (request->offset > sizeof(request->header_value) - 1) {

From 07b2697ae387e3157616e606ed970291d487270c Mon Sep 17 00:00:00 2001
From: Scott Shawcroft 
Date: Fri, 1 Jul 2022 16:57:10 -0700
Subject: [PATCH 02/10] WIP websocket to serial

---
 ports/espressif/common-hal/hashlib/Hash.c     |   5 +
 py/circuitpy_mpconfig.mk                      |   3 +
 supervisor/shared/serial.c                    |  27 +++
 supervisor/shared/web_workflow/web_workflow.c |  41 +++-
 supervisor/shared/web_workflow/websocket.c    | 179 ++++++++++++++++++
 supervisor/shared/web_workflow/websocket.h    |  38 ++++
 supervisor/supervisor.mk                      |   4 +-
 7 files changed, 288 insertions(+), 9 deletions(-)
 create mode 100644 supervisor/shared/web_workflow/websocket.c
 create mode 100644 supervisor/shared/web_workflow/websocket.h

diff --git a/ports/espressif/common-hal/hashlib/Hash.c b/ports/espressif/common-hal/hashlib/Hash.c
index 8e2ab02edb..8090128acb 100644
--- a/ports/espressif/common-hal/hashlib/Hash.c
+++ b/ports/espressif/common-hal/hashlib/Hash.c
@@ -40,7 +40,12 @@ void common_hal_hashlib_hash_digest(hashlib_hash_obj_t *self, uint8_t *data, siz
         return;
     }
     if (self->hash_type == MBEDTLS_SSL_HASH_SHA1) {
+        // We copy the sha1 state so we can continue to update if needed or get
+        // the digest a second time.
+        mbedtls_sha1_context copy;
+        mbedtls_sha1_clone(©, &self->sha1);
         mbedtls_sha1_finish_ret(&self->sha1, data);
+        mbedtls_sha1_clone(&self->sha1, ©);
     }
 }
 
diff --git a/py/circuitpy_mpconfig.mk b/py/circuitpy_mpconfig.mk
index 8505fe1cb7..de88dcc727 100644
--- a/py/circuitpy_mpconfig.mk
+++ b/py/circuitpy_mpconfig.mk
@@ -242,6 +242,9 @@ CFLAGS += -DCIRCUITPY_GIFIO=$(CIRCUITPY_GIFIO)
 CIRCUITPY_GNSS ?= 0
 CFLAGS += -DCIRCUITPY_GNSS=$(CIRCUITPY_GNSS)
 
+CIRCUITPY_HASHLIB ?= $(CIRCUITPY_WEB_WORKFLOW)
+CFLAGS += -DCIRCUITPY_HASHLIB=$(CIRCUITPY_HASHLIB)
+
 CIRCUITPY_I2CPERIPHERAL ?= $(CIRCUITPY_FULL_BUILD)
 CFLAGS += -DCIRCUITPY_I2CPERIPHERAL=$(CIRCUITPY_I2CPERIPHERAL)
 
diff --git a/supervisor/shared/serial.c b/supervisor/shared/serial.c
index af90fce4d7..04e974e4b5 100644
--- a/supervisor/shared/serial.c
+++ b/supervisor/shared/serial.c
@@ -45,6 +45,10 @@
 #include "tusb.h"
 #endif
 
+#if CIRCUITPY_WEB_WORKFLOW
+#include "supervisor/shared/web_workflow/websocket.h"
+#endif
+
 /*
  * Note: DEBUG_UART currently only works on STM32 and nRF.
  * Enabling on another platform will cause a crash.
@@ -165,6 +169,13 @@ bool serial_connected(void) {
     }
     #endif
 
+    #if CIRCUITPY_WEB_WORKFLOW
+    if (websocket_connected()) {
+        return true;
+    }
+    #endif
+
+
     if (port_serial_connected()) {
         return true;
     }
@@ -195,6 +206,12 @@ char serial_read(void) {
     }
     #endif
 
+    #if CIRCUITPY_WEB_WORKFLOW
+    if (websocket_available()) {
+        return websocket_read_char();
+    }
+    #endif
+
     #if CIRCUITPY_USB_CDC
     if (!usb_cdc_console_enabled()) {
         return -1;
@@ -229,6 +246,12 @@ bool serial_bytes_available(void) {
     }
     #endif
 
+    #if CIRCUITPY_WEB_WORKFLOW
+    if (websocket_available()) {
+        return true;
+    }
+    #endif
+
     #if CIRCUITPY_USB_CDC
     if (usb_cdc_console_enabled() && tud_cdc_available() > 0) {
         return true;
@@ -271,6 +294,10 @@ void serial_write_substring(const char *text, uint32_t length) {
     ble_serial_write(text, length);
     #endif
 
+    #if CIRCUITPY_WEB_WORKFLOW
+    websocket_write(text, length);
+    #endif
+
     #if CIRCUITPY_USB_CDC
     if (!usb_cdc_console_enabled()) {
         return;
diff --git a/supervisor/shared/web_workflow/web_workflow.c b/supervisor/shared/web_workflow/web_workflow.c
index b9a1de67f2..07298c17e9 100644
--- a/supervisor/shared/web_workflow/web_workflow.c
+++ b/supervisor/shared/web_workflow/web_workflow.c
@@ -39,8 +39,11 @@
 #include "supervisor/shared/reload.h"
 #include "supervisor/shared/translate/translate.h"
 #include "supervisor/shared/web_workflow/web_workflow.h"
+#include "supervisor/shared/web_workflow/websocket.h"
 #include "supervisor/usb.h"
 
+#include "shared-bindings/hashlib/__init__.h"
+#include "shared-bindings/hashlib/Hash.h"
 #include "shared-bindings/mdns/RemoteService.h"
 #include "shared-bindings/mdns/Server.h"
 #include "shared-bindings/socketpool/__init__.h"
@@ -260,20 +263,19 @@ void supervisor_start_web_workflow(void) {
     active.num = -1;
     active.connected = false;
 
+    websocket_init();
+
     // TODO:
     // GET /cp/serial.txt
     //   - Most recent 1k of serial output.
     // GET /edit/
     //   - Super basic editor
-    // GET /ws/circuitpython
-    // GET /ws/user
-    //   - WebSockets
     #endif
 }
 
 static void _send_raw(socketpool_socket_obj_t *socket, const uint8_t *buf, int len) {
     int sent = -EAGAIN;
-    while (sent == -EAGAIN) {
+    while (sent == -EAGAIN && common_hal_socketpool_socket_get_connected(socket)) {
         sent = socketpool_socket_send(socket, buf, len);
     }
     if (sent < len) {
@@ -851,17 +853,39 @@ static void _reply_static(socketpool_socket_obj_t *socket, _request *request, co
 
 #define _REPLY_STATIC(socket, request, filename) _reply_static(socket, request, filename, filename##_length, filename##_content_type)
 
+
+
 static void _reply_websocket_upgrade(socketpool_socket_obj_t *socket, _request *request) {
     ESP_LOGI(TAG, "websocket!");
     // Compute accept key
+    hashlib_hash_obj_t hash;
+    common_hal_hashlib_new(&hash, "sha1");
+    common_hal_hashlib_hash_update(&hash, (const uint8_t *)request->websocket_key, strlen(request->websocket_key));
+    const char *magic_string = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11";
+    common_hal_hashlib_hash_update(&hash, (const uint8_t *)magic_string, strlen(magic_string));
+    size_t digest_size = common_hal_hashlib_hash_get_digest_size(&hash);
+    size_t encoded_size = (digest_size + 1) * 4 / 3 + 1;
+    uint8_t encoded_accept[encoded_size];
+    common_hal_hashlib_hash_digest(&hash, encoded_accept, sizeof(encoded_accept));
+    _base64_in_place((char *)encoded_accept, digest_size, encoded_size);
+
     // Reply with upgrade
-    // Copy socket state into websocket and mark given socket as closed even though it isn't actually.
+    _send_strs(socket, "HTTP/1.1 101 Switching Protocols\r\n",
+        "Upgrade: websocket\r\n",
+        "Connection: Upgrade\r\n",
+        "Sec-WebSocket-Accept: ", encoded_accept, "\r\n",
+        "\r\n", NULL);
+    websocket_handoff(socket);
+
+    ESP_LOGI(TAG, "socket upgrade done");
+    // socket is now closed and "disconnected".
 }
 
 static bool _reply(socketpool_socket_obj_t *socket, _request *request) {
     if (request->redirect) {
         _reply_redirect(socket, request, request->path);
     } else if (strlen(request->origin) > 0 && !_origin_ok(request->origin)) {
+        ESP_LOGI(TAG, "bad origin %s", request->origin);
         _reply_forbidden(socket, request);
     } else if (memcmp(request->path, "/fs/", 4) == 0) {
         if (!request->authenticated) {
@@ -995,7 +1019,7 @@ static bool _reply(socketpool_socket_obj_t *socket, _request *request) {
         } else if (strcmp(path, "/version.json") == 0) {
             _reply_with_version_json(socket, request);
         } else if (strcmp(path, "/serial/") == 0) {
-            if (!request->authenticated) {
+            if (false && !request->authenticated) {
                 if (_api_password[0] != '\0') {
                     _reply_unauthorized(socket, request);
                 } else {
@@ -1007,7 +1031,6 @@ static bool _reply(socketpool_socket_obj_t *socket, _request *request) {
             } else {
                 _REPLY_STATIC(socket, request, serial_html);
             }
-            _reply_with_version_json(socket, request);
         } else {
             _reply_missing(socket, request);
         }
@@ -1177,6 +1200,7 @@ static void _process_request(socketpool_socket_obj_t *socket, _request *request)
         return;
     }
     bool reload = _reply(socket, request);
+    ESP_LOGI(TAG, "reply done");
     _reset_request(request);
     autoreload_resume(AUTORELOAD_SUSPEND_WEB);
     if (reload) {
@@ -1194,10 +1218,12 @@ void supervisor_web_workflow_background(void) {
         uint32_t port;
         int newsoc = socketpool_socket_accept(&listening, (uint8_t *)&ip, &port);
         if (newsoc == -EBADF) {
+            ESP_LOGI(TAG, "listen closed");
             common_hal_socketpool_socket_close(&listening);
             return;
         }
         if (newsoc > 0) {
+            ESP_LOGI(TAG, "new socket %d", newsoc);
             // Close the active socket because we have another we accepted.
             if (!common_hal_socketpool_socket_get_closed(&active)) {
                 common_hal_socketpool_socket_close(&active);
@@ -1218,6 +1244,7 @@ void supervisor_web_workflow_background(void) {
 
     // If we have a request in progress, continue working on it.
     if (common_hal_socketpool_socket_get_connected(&active)) {
+        ESP_LOGI(TAG, "active connected");
         _process_request(&active, &active_request);
     }
 }
diff --git a/supervisor/shared/web_workflow/websocket.c b/supervisor/shared/web_workflow/websocket.c
new file mode 100644
index 0000000000..8aec9d206c
--- /dev/null
+++ b/supervisor/shared/web_workflow/websocket.c
@@ -0,0 +1,179 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2022 Scott Shawcroft for Adafruit Industries
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "supervisor/shared/web_workflow/websocket.h"
+
+typedef struct {
+    socketpool_socket_obj_t socket;
+    uint8_t opcode;
+    uint8_t frame_len;
+    uint8_t payload_len_size;
+    bool masked;
+    uint32_t mask;
+    size_t frame_index;
+    size_t payload_remaining;
+} _websocket;
+
+static _websocket cp_serial;
+
+static const char *TAG = "CP websocket";
+
+void websocket_init(void) {
+    cp_serial.socket.num = -1;
+    cp_serial.socket.connected = false;
+}
+
+void websocket_handoff(socketpool_socket_obj_t *socket) {
+    ESP_LOGI(TAG, "socket handed off");
+    cp_serial.socket = *socket;
+    // Mark the original socket object as closed without telling the lower level.
+    socket->connected = false;
+    socket->num = -1;
+    ESP_LOGI(TAG, "socket hand off done");
+}
+
+bool websocket_connected(void) {
+    return common_hal_socketpool_socket_get_connected(&cp_serial.socket);
+}
+
+static bool _read_byte(uint8_t *c) {
+    int len = socketpool_socket_recv_into(&cp_serial.socket, c, 1);
+    if (len != 1) {
+        if (len != -EAGAIN) {
+            ESP_LOGE(TAG, "recv error %d", len);
+        }
+        return false;
+    }
+    return true;
+}
+
+static void _read_next_frame_header(void) {
+    uint8_t h;
+    if (cp_serial.frame_index == 0 && _read_byte(&h)) {
+        cp_serial.frame_index++;
+        cp_serial.opcode = h & 0xf;
+        ESP_LOGI(TAG, "fin %d opcode %x", h >> 7, cp_serial.opcode);
+    }
+    if (cp_serial.frame_index == 1 && _read_byte(&h)) {
+        cp_serial.frame_index++;
+        uint8_t len = h & 0xf;
+        cp_serial.masked = (h >> 7) == 1;
+        if (len <= 125) {
+            cp_serial.payload_remaining = len;
+            cp_serial.payload_len_size = 0;
+        } else if (len == 126) { // 16 bit length
+            cp_serial.payload_len_size = 2;
+        } else if (len == 127) { // 64 bit length
+            cp_serial.payload_len_size = 8;
+        }
+        cp_serial.frame_len = 2 + cp_serial.payload_len_size;
+        if (cp_serial.masked) {
+            cp_serial.frame_len += 4;
+        }
+
+        ESP_LOGI(TAG, "mask %d length %x", cp_serial.masked, len);
+    }
+    while (cp_serial.frame_index > 1 &&
+           cp_serial.frame_index < (cp_serial.payload_len_size + 2) &&
+           _read_byte(&h)) {
+        cp_serial.frame_index++;
+        cp_serial.payload_remaining = cp_serial.payload_remaining << 8 | c;
+    }
+    while (cp_serial.frame_index > (cp_serial.payload_len_size + 2) &&
+           cp_serial.frame_index < cp_serial.frame_len &&
+           _read_byte(&h)) {
+        cp_serial.frame_index++;
+        cp_serial.mask = cp_serial.mask << 8 | c;
+    }
+}
+
+static bool _read_next_payload_byte(uint8_t *c) {
+    _read_next_frame_header();
+    if (cp_serial.frame_index > cp_serial.frame_len &&
+        cp_serial.payload_remaining > 0) {
+        if (_read_byte(c)) {
+            cp_serial.frame_index++;
+            cp_serial.payload_remaining--;
+            return true;
+        }
+    }
+    return false;
+}
+
+bool websocket_available(void) {
+    if (!websocket_connected()) {
+        return false;
+    }
+    _read_next_frame_header();
+    return cp_serial.payload_remaining > 0 && cp_serial.frame_index >= cp_serial.frame_len;
+}
+
+char websocket_read_char(void) {
+    return _read_next_payload_byte();
+}
+
+static void _send_raw(socketpool_socket_obj_t *socket, const uint8_t *buf, int len) {
+    int sent = -EAGAIN;
+    while (sent == -EAGAIN) {
+        sent = socketpool_socket_send(socket, buf, len);
+    }
+    if (sent < len) {
+        ESP_LOGE(TAG, "short send %d %d", sent, len);
+    }
+}
+
+static void _websocket_send(_websocket *ws, const char *text, size_t len) {
+    if (!websocket_connected()) {
+        return;
+    }
+    uint32_t opcode = 1;
+    uint8_t frame_header[2];
+    frame_header[0] = 1 << 7 | opcode;
+    uint8_t payload_len;
+    if (len <= 125) {
+        payload_len = len;
+    } else if (len < (1 << 16)) {
+        payload_len = 126;
+    } else {
+        payload_len = 127;
+    }
+    frame_header[1] = payload_len;
+    _send_raw(&ws->socket, (const uint8_t *)frame_header, 2);
+    if (payload_len == 126) {
+        _send_raw(&ws->socket, (const uint8_t *)&len, 2);
+    } else if (payload_len == 127) {
+        uint32_t zero = 0;
+        // 64 bits where top four bytes are zero.
+        _send_raw(&ws->socket, (const uint8_t *)&zero, 4);
+        _send_raw(&ws->socket, (const uint8_t *)&len, 4);
+    }
+    _send_raw(&ws->socket, (const uint8_t *)text, len);
+    ESP_LOGI(TAG, "sent over websocket: %s", text);
+}
+
+void websocket_write(const char *text, size_t len) {
+    _websocket_send(&cp_serial, text, len);
+}
diff --git a/supervisor/shared/web_workflow/websocket.h b/supervisor/shared/web_workflow/websocket.h
new file mode 100644
index 0000000000..c5c5114586
--- /dev/null
+++ b/supervisor/shared/web_workflow/websocket.h
@@ -0,0 +1,38 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2022 Scott Shawcroft for Adafruit Industries
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#pragma once
+
+#include 
+
+#include "shared-bindings/socketpool/Socket.h"
+
+void websocket_init(void);
+void websocket_handoff(socketpool_socket_obj_t *socket);
+bool websocket_connected(void);
+bool websocket_available(void);
+char websocket_read_char(void);
+void websocket_write(const char *text, size_t len);
diff --git a/supervisor/supervisor.mk b/supervisor/supervisor.mk
index 45df5f687f..5b8cb513a7 100644
--- a/supervisor/supervisor.mk
+++ b/supervisor/supervisor.mk
@@ -168,8 +168,8 @@ $(BUILD)/autogen_web_workflow_static.c: ../../tools/gen_web_workflow_static.py $
 		$(STATIC_RESOURCES)
 
 ifeq ($(CIRCUITPY_WEB_WORKFLOW),1)
-  SRC_SUPERVISOR += supervisor/shared/web_workflow/web_workflow.c
-
+  SRC_SUPERVISOR += supervisor/shared/web_workflow/web_workflow.c \
+                    supervisor/shared/web_workflow/websocket.c
   SRC_SUPERVISOR += $(BUILD)/autogen_web_workflow_static.c
 endif
 

From a4035aa1f73ea1033743d918948d80a71b88dab5 Mon Sep 17 00:00:00 2001
From: Scott Shawcroft 
Date: Wed, 6 Jul 2022 17:05:14 -0700
Subject: [PATCH 03/10] websocket serial kinda works

---
 .../shared/web_workflow/static/serial.js      | 10 +++---
 supervisor/shared/web_workflow/web_workflow.c |  3 +-
 supervisor/shared/web_workflow/websocket.c    | 34 ++++++++++++++-----
 3 files changed, 32 insertions(+), 15 deletions(-)

diff --git a/supervisor/shared/web_workflow/static/serial.js b/supervisor/shared/web_workflow/static/serial.js
index 36c980dc08..5fb80b59fa 100644
--- a/supervisor/shared/web_workflow/static/serial.js
+++ b/supervisor/shared/web_workflow/static/serial.js
@@ -5,7 +5,7 @@ function onSubmit() {
   var input = document.getElementById("input");
   // You can send message to the Web Socket using ws.send.
   ws.send(input.value);
-  output("send: " + input.value);
+  // output("send: " + input.value);
   input.value = "";
   input.focus();
 }
@@ -25,19 +25,19 @@ ws = new WebSocket("ws://cpy-f57ce8.local/cp/serial/");
 
 // Set event handlers.
 ws.onopen = function() {
-  output("onopen");
+  console.log("onopen");
 };
 
 ws.onmessage = function(e) {
   // e.data contains received string.
-  output("onmessage: " + e.data);
+  output(e.data);
 };
 
 ws.onclose = function() {
-  output("onclose");
+  console.log("onclose");
 };
 
 ws.onerror = function(e) {
-  output("onerror");
+  // output("onerror");
   console.log(e)
 };
diff --git a/supervisor/shared/web_workflow/web_workflow.c b/supervisor/shared/web_workflow/web_workflow.c
index 07298c17e9..c7bf673fdc 100644
--- a/supervisor/shared/web_workflow/web_workflow.c
+++ b/supervisor/shared/web_workflow/web_workflow.c
@@ -1059,6 +1059,7 @@ static bool _reply(socketpool_socket_obj_t *socket, _request *request) {
 }
 
 static void _reset_request(_request *request) {
+    ESP_LOGI(TAG, "reset request");
     request->state = STATE_METHOD;
     request->origin[0] = '\0';
     request->content_length = 0;
@@ -1244,7 +1245,7 @@ void supervisor_web_workflow_background(void) {
 
     // If we have a request in progress, continue working on it.
     if (common_hal_socketpool_socket_get_connected(&active)) {
-        ESP_LOGI(TAG, "active connected");
+        // ESP_LOGI(TAG, "active connected %d", active_request.in_progress);
         _process_request(&active, &active_request);
     }
 }
diff --git a/supervisor/shared/web_workflow/websocket.c b/supervisor/shared/web_workflow/websocket.c
index 8aec9d206c..446f2073c3 100644
--- a/supervisor/shared/web_workflow/websocket.c
+++ b/supervisor/shared/web_workflow/websocket.c
@@ -32,8 +32,8 @@ typedef struct {
     uint8_t frame_len;
     uint8_t payload_len_size;
     bool masked;
-    uint32_t mask;
-    size_t frame_index;
+    uint8_t mask[4];
+    int frame_index;
     size_t payload_remaining;
 } _websocket;
 
@@ -96,27 +96,38 @@ static void _read_next_frame_header(void) {
 
         ESP_LOGI(TAG, "mask %d length %x", cp_serial.masked, len);
     }
-    while (cp_serial.frame_index > 1 &&
+    while (cp_serial.frame_index >= 2 &&
            cp_serial.frame_index < (cp_serial.payload_len_size + 2) &&
            _read_byte(&h)) {
         cp_serial.frame_index++;
-        cp_serial.payload_remaining = cp_serial.payload_remaining << 8 | c;
+        cp_serial.payload_remaining = cp_serial.payload_remaining << 8 | h;
     }
-    while (cp_serial.frame_index > (cp_serial.payload_len_size + 2) &&
+    int mask_start = cp_serial.payload_len_size + 2;
+    while (cp_serial.frame_index >= mask_start &&
            cp_serial.frame_index < cp_serial.frame_len &&
            _read_byte(&h)) {
+        size_t mask_offset = cp_serial.frame_index - mask_start;
+        cp_serial.mask[mask_offset] = h;
         cp_serial.frame_index++;
-        cp_serial.mask = cp_serial.mask << 8 | c;
+        ESP_LOGI(TAG, "mask %08x", (uint32_t)*cp_serial.mask);
     }
 }
 
 static bool _read_next_payload_byte(uint8_t *c) {
     _read_next_frame_header();
-    if (cp_serial.frame_index > cp_serial.frame_len &&
+    if (cp_serial.frame_index >= cp_serial.frame_len &&
         cp_serial.payload_remaining > 0) {
         if (_read_byte(c)) {
+            uint8_t mask_offset = (cp_serial.frame_index - cp_serial.frame_len) % 4;
+            ESP_LOGI(TAG, "payload byte read %02x mask offset %d", *c, mask_offset);
+            *c ^= cp_serial.mask[mask_offset];
+            ESP_LOGI(TAG, "byte unmasked %02x", *c);
             cp_serial.frame_index++;
             cp_serial.payload_remaining--;
+            ESP_LOGI(TAG, "payload remaining %d", cp_serial.payload_remaining);
+            if (cp_serial.payload_remaining == 0) {
+                cp_serial.frame_index = 0;
+            }
             return true;
         }
     }
@@ -132,7 +143,9 @@ bool websocket_available(void) {
 }
 
 char websocket_read_char(void) {
-    return _read_next_payload_byte();
+    uint8_t c;
+    _read_next_payload_byte(&c);
+    return c;
 }
 
 static void _send_raw(socketpool_socket_obj_t *socket, const uint8_t *buf, int len) {
@@ -171,7 +184,10 @@ static void _websocket_send(_websocket *ws, const char *text, size_t len) {
         _send_raw(&ws->socket, (const uint8_t *)&len, 4);
     }
     _send_raw(&ws->socket, (const uint8_t *)text, len);
-    ESP_LOGI(TAG, "sent over websocket: %s", text);
+    char copy[len];
+    memcpy(copy, text, len);
+    copy[len] = '\0';
+    ESP_LOGI(TAG, "sent over websocket: %s", copy);
 }
 
 void websocket_write(const char *text, size_t len) {

From 8d9c9952981d5c8eb10f7db2da587a8aabc360bd Mon Sep 17 00:00:00 2001
From: Scott Shawcroft 
Date: Thu, 7 Jul 2022 16:55:04 -0700
Subject: [PATCH 04/10] Support ping & close. Refine html and js for serial

---
 .../shared/web_workflow/static/serial.html    |  8 +--
 .../shared/web_workflow/static/serial.js      | 36 +++++++++-
 supervisor/shared/web_workflow/websocket.c    | 68 +++++++++++++++----
 3 files changed, 89 insertions(+), 23 deletions(-)

diff --git a/supervisor/shared/web_workflow/static/serial.html b/supervisor/shared/web_workflow/static/serial.html
index 766a7a23e2..d8ab86a7c8 100644
--- a/supervisor/shared/web_workflow/static/serial.html
+++ b/supervisor/shared/web_workflow/static/serial.html
@@ -7,10 +7,8 @@
 
 
   

-  

-    
-    
-    
-  

+  
+  
+  
 
 
diff --git a/supervisor/shared/web_workflow/static/serial.js b/supervisor/shared/web_workflow/static/serial.js
index 5fb80b59fa..ab0893ad08 100644
--- a/supervisor/shared/web_workflow/static/serial.js
+++ b/supervisor/shared/web_workflow/static/serial.js
@@ -1,8 +1,20 @@
 
 var ws;
+var input = document.getElementById("input");
+var title = document.querySelector("title");
+
+function set_enabled(enabled) {
+  input.disabled = !enabled;
+  var buttons = document.querySelectorAll("button");
+  for (button of buttons) {
+    button.disabled = !enabled;
+  }
+}
+
+set_enabled(false);
 
 function onSubmit() {
-  var input = document.getElementById("input");
+  console.log("submit");
   // You can send message to the Web Socket using ws.send.
   ws.send(input.value);
   // output("send: " + input.value);
@@ -11,6 +23,7 @@ function onSubmit() {
 }
 
 function onCloseClick() {
+  console.log("close clicked");
   ws.close();
 }
 
@@ -26,18 +39,35 @@ ws = new WebSocket("ws://cpy-f57ce8.local/cp/serial/");
 // Set event handlers.
 ws.onopen = function() {
   console.log("onopen");
+  set_enabled(true);
 };
 
+var setting_title = false;
 ws.onmessage = function(e) {
   // e.data contains received string.
-  output(e.data);
+  if (e.data == "\x1b]0;") {
+    setting_title = true;
+    title.textContent = "";
+  } else if (e.data == "\x1b\\") {
+    setting_title = false;
+  } else if (setting_title) {
+    title.textContent += e.data;
+  } else {
+    output(e.data);
+  }
 };
 
 ws.onclose = function() {
   console.log("onclose");
+  set_enabled(false);
 };
 
 ws.onerror = function(e) {
   // output("onerror");
-  console.log(e)
+  console.log(e);
+  set_enabled(false);
 };
+
+input.onbeforeinput = function(e) {
+  console.log(e);
+}
diff --git a/supervisor/shared/web_workflow/websocket.c b/supervisor/shared/web_workflow/websocket.c
index 446f2073c3..0f4c4fc84c 100644
--- a/supervisor/shared/web_workflow/websocket.c
+++ b/supervisor/shared/web_workflow/websocket.c
@@ -32,6 +32,7 @@ typedef struct {
     uint8_t frame_len;
     uint8_t payload_len_size;
     bool masked;
+    bool closed;
     uint8_t mask[4];
     int frame_index;
     size_t payload_remaining;
@@ -49,6 +50,10 @@ void websocket_init(void) {
 void websocket_handoff(socketpool_socket_obj_t *socket) {
     ESP_LOGI(TAG, "socket handed off");
     cp_serial.socket = *socket;
+    cp_serial.closed = false;
+    cp_serial.opcode = 0;
+    cp_serial.frame_index = 0;
+    cp_serial.frame_len = 2;
     // Mark the original socket object as closed without telling the lower level.
     socket->connected = false;
     socket->num = -1;
@@ -56,7 +61,7 @@ void websocket_handoff(socketpool_socket_obj_t *socket) {
 }
 
 bool websocket_connected(void) {
-    return common_hal_socketpool_socket_get_connected(&cp_serial.socket);
+    return !cp_serial.closed && common_hal_socketpool_socket_get_connected(&cp_serial.socket);
 }
 
 static bool _read_byte(uint8_t *c) {
@@ -70,6 +75,16 @@ static bool _read_byte(uint8_t *c) {
     return true;
 }
 
+static void _send_raw(socketpool_socket_obj_t *socket, const uint8_t *buf, int len) {
+    int sent = -EAGAIN;
+    while (sent == -EAGAIN) {
+        sent = socketpool_socket_send(socket, buf, len);
+    }
+    if (sent < len) {
+        ESP_LOGE(TAG, "short send on %d err %d len %d", socket->num, sent, len);
+    }
+}
+
 static void _read_next_frame_header(void) {
     uint8_t h;
     if (cp_serial.frame_index == 0 && _read_byte(&h)) {
@@ -111,20 +126,53 @@ static void _read_next_frame_header(void) {
         cp_serial.frame_index++;
         ESP_LOGI(TAG, "mask %08x", (uint32_t)*cp_serial.mask);
     }
+    // Reply to PINGs and CLOSE.
+    while ((cp_serial.opcode == 0x8 ||
+            cp_serial.opcode == 0x9) &&
+           cp_serial.frame_index >= cp_serial.frame_len) {
+
+        if (cp_serial.frame_index == cp_serial.frame_len) {
+            uint8_t opcode = 0x8; // CLOSE
+            if (cp_serial.opcode == 0x9) {
+                ESP_LOGI(TAG, "websocket ping");
+                opcode = 0xA; // PONG
+            }
+            uint8_t frame_header[2];
+            frame_header[0] = 1 << 7 | opcode;
+            if (cp_serial.payload_remaining > 125) {
+                ESP_LOGE(TAG, "CLOSE or PING has long payload");
+            }
+            frame_header[1] = cp_serial.payload_remaining;
+            _send_raw(&cp_serial.socket, (const uint8_t *)frame_header, 2);
+        }
+
+        if (cp_serial.payload_remaining > 0 && _read_byte(&h)) {
+            // Send the payload back to the client.
+            cp_serial.frame_index++;
+            cp_serial.payload_remaining--;
+            _send_raw(&cp_serial.socket, &h, 1);
+        }
+
+        if (cp_serial.payload_remaining == 0) {
+            cp_serial.frame_index = 0;
+            if (cp_serial.opcode == 0x8) {
+                ESP_LOGI(TAG, "websocket closed");
+                cp_serial.closed = true;
+            }
+        }
+    }
 }
 
 static bool _read_next_payload_byte(uint8_t *c) {
     _read_next_frame_header();
-    if (cp_serial.frame_index >= cp_serial.frame_len &&
+    if (cp_serial.opcode == 0x1 &&
+        cp_serial.frame_index >= cp_serial.frame_len &&
         cp_serial.payload_remaining > 0) {
         if (_read_byte(c)) {
             uint8_t mask_offset = (cp_serial.frame_index - cp_serial.frame_len) % 4;
-            ESP_LOGI(TAG, "payload byte read %02x mask offset %d", *c, mask_offset);
             *c ^= cp_serial.mask[mask_offset];
-            ESP_LOGI(TAG, "byte unmasked %02x", *c);
             cp_serial.frame_index++;
             cp_serial.payload_remaining--;
-            ESP_LOGI(TAG, "payload remaining %d", cp_serial.payload_remaining);
             if (cp_serial.payload_remaining == 0) {
                 cp_serial.frame_index = 0;
             }
@@ -148,16 +196,6 @@ char websocket_read_char(void) {
     return c;
 }
 
-static void _send_raw(socketpool_socket_obj_t *socket, const uint8_t *buf, int len) {
-    int sent = -EAGAIN;
-    while (sent == -EAGAIN) {
-        sent = socketpool_socket_send(socket, buf, len);
-    }
-    if (sent < len) {
-        ESP_LOGE(TAG, "short send %d %d", sent, len);
-    }
-}
-
 static void _websocket_send(_websocket *ws, const char *text, size_t len) {
     if (!websocket_connected()) {
         return;

From 557e35469f7bece08c68a65a29dca82126c4b242 Mon Sep 17 00:00:00 2001
From: Scott Shawcroft 
Date: Fri, 8 Jul 2022 16:57:19 -0700
Subject: [PATCH 05/10] Make serial page work ok including on mobile

---
 .../shared/web_workflow/static/serial.html    | 22 +++++--
 .../shared/web_workflow/static/serial.js      | 60 +++++++++++--------
 supervisor/shared/web_workflow/websocket.c    |  9 ++-
 3 files changed, 59 insertions(+), 32 deletions(-)

diff --git a/supervisor/shared/web_workflow/static/serial.html b/supervisor/shared/web_workflow/static/serial.html
index d8ab86a7c8..aaf416b8b3 100644
--- a/supervisor/shared/web_workflow/static/serial.html
+++ b/supervisor/shared/web_workflow/static/serial.html
@@ -1,14 +1,24 @@
 
-
+
+
 
   Simple client
 
   
 
-
-  

-  
-  
-  
+
+  

+    

+     
+  

+  

+    

+      Ctrl
+      
+      
+    

+    
+    
+  

 
 
diff --git a/supervisor/shared/web_workflow/static/serial.js b/supervisor/shared/web_workflow/static/serial.js
index ab0893ad08..595bbe6653 100644
--- a/supervisor/shared/web_workflow/static/serial.js
+++ b/supervisor/shared/web_workflow/static/serial.js
@@ -1,7 +1,9 @@
 
 var ws;
 var input = document.getElementById("input");
+input.value = "";
 var title = document.querySelector("title");
+var log = document.getElementById("log");
 
 function set_enabled(enabled) {
   input.disabled = !enabled;
@@ -14,35 +16,21 @@ function set_enabled(enabled) {
 set_enabled(false);
 
 function onSubmit() {
-  console.log("submit");
-  // You can send message to the Web Socket using ws.send.
-  ws.send(input.value);
-  // output("send: " + input.value);
-  input.value = "";
-  input.focus();
-}
-
-function onCloseClick() {
-  console.log("close clicked");
-  ws.close();
-}
-
-function output(str) {
-  var log = document.getElementById("log");
-  log.innerHTML += str;
+    ws.send("\r");
+    input.value = "";
+    input.focus();
 }
 
 // Connect to Web Socket
-ws = new WebSocket("ws://cpy-f57ce8.local/cp/serial/");
-// ws = new WebSocket("ws://127.0.0.1:9001")
+ws = new WebSocket("ws://" + window.location.host + "/cp/serial/");
 
-// Set event handlers.
 ws.onopen = function() {
-  console.log("onopen");
   set_enabled(true);
 };
 
 var setting_title = false;
+var encoder = new TextEncoder();
+var left_count = 0;
 ws.onmessage = function(e) {
   // e.data contains received string.
   if (e.data == "\x1b]0;") {
@@ -52,22 +40,44 @@ ws.onmessage = function(e) {
     setting_title = false;
   } else if (setting_title) {
     title.textContent += e.data;
+  } else if (e.data == "\b") {
+    left_count += 1;
+  } else if (e.data == "\x1b[K") { // Clear line
+    log.textContent = log.textContent.slice(0, -left_count);
+    left_count = 0;
   } else {
-    output(e.data);
+    log.textContent += e.data;
   }
+  document.querySelector("span").scrollIntoView();
 };
 
 ws.onclose = function() {
-  console.log("onclose");
   set_enabled(false);
 };
 
 ws.onerror = function(e) {
-  // output("onerror");
-  console.log(e);
   set_enabled(false);
 };
 
 input.onbeforeinput = function(e) {
-  console.log(e);
+  if (e.inputType == "insertLineBreak") {
+    ws.send("\r");
+    input.value = "";
+    input.focus();
+    e.preventDefault();
+  } else if (e.inputType == "insertText") {
+    ws.send(e.data);
+  } else if (e.inputType == "deleteContentBackward") {
+    ws.send("\b");
+  }
+}
+
+let ctrl_c = document.querySelector("#c");
+ctrl_c.onclick = function() {
+  ws.send("\x03");
+}
+
+let ctrl_d = document.querySelector("#d");
+ctrl_d.onclick = function() {
+  ws.send("\x04");
 }
diff --git a/supervisor/shared/web_workflow/websocket.c b/supervisor/shared/web_workflow/websocket.c
index 0f4c4fc84c..7a2f37b168 100644
--- a/supervisor/shared/web_workflow/websocket.c
+++ b/supervisor/shared/web_workflow/websocket.c
@@ -124,7 +124,6 @@ static void _read_next_frame_header(void) {
         size_t mask_offset = cp_serial.frame_index - mask_start;
         cp_serial.mask[mask_offset] = h;
         cp_serial.frame_index++;
-        ESP_LOGI(TAG, "mask %08x", (uint32_t)*cp_serial.mask);
     }
     // Reply to PINGs and CLOSE.
     while ((cp_serial.opcode == 0x8 ||
@@ -136,6 +135,11 @@ static void _read_next_frame_header(void) {
             if (cp_serial.opcode == 0x9) {
                 ESP_LOGI(TAG, "websocket ping");
                 opcode = 0xA; // PONG
+            } else {
+                // Set the TCP socket to send immediately so that we send the payload back before
+                // closing the connection.
+                int nodelay = 1;
+                lwip_setsockopt(cp_serial.socket.num, IPPROTO_TCP, TCP_NODELAY, &nodelay, sizeof(nodelay));
             }
             uint8_t frame_header[2];
             frame_header[0] = 1 << 7 | opcode;
@@ -158,6 +162,8 @@ static void _read_next_frame_header(void) {
             if (cp_serial.opcode == 0x8) {
                 ESP_LOGI(TAG, "websocket closed");
                 cp_serial.closed = true;
+
+                common_hal_socketpool_socket_close(&cp_serial.socket);
             }
         }
     }
@@ -193,6 +199,7 @@ bool websocket_available(void) {
 char websocket_read_char(void) {
     uint8_t c;
     _read_next_payload_byte(&c);
+    ESP_LOGI(TAG, "read %c", c);
     return c;
 }
 

From 8cfdfb95f78a8f5082fbdefcd0c8adbf5cb49cc6 Mon Sep 17 00:00:00 2001
From: Scott Shawcroft 
Date: Mon, 11 Jul 2022 14:32:28 -0700
Subject: [PATCH 06/10] Remove extra logging, auth /cp/serial and add doc

---
 docs/workflows.md                             | 78 +++++++++++--------
 .../shared/web_workflow/static/welcome.html   |  3 +-
 supervisor/shared/web_workflow/web_workflow.c |  4 +-
 supervisor/shared/web_workflow/websocket.c    |  9 ---
 4 files changed, 50 insertions(+), 44 deletions(-)

diff --git a/docs/workflows.md b/docs/workflows.md
index 1b262011b2..b011acd01c 100644
--- a/docs/workflows.md
+++ b/docs/workflows.md
@@ -286,6 +286,44 @@ not protected by basic auth in case the device is someone elses.
 
 Only `GET` requests are supported and will return `405 Method Not Allowed` otherwise.
 
+#### `/cp/devices.json`
+
+Returns information about other devices found on the network using MDNS.
+
+* `total`: Total MDNS response count. May be more than in `devices` if internal limits were hit.
+* `devices`: List of discovered devices.
+	* `hostname`: MDNS hostname
+	* `instance_name`: MDNS instance name. Defaults to human readable board name.
+	* `port`: Port of CircuitPython Web API
+	* `ip`: IP address
+
+Example:
+```sh
+curl -v -L http://circuitpython.local/cp/devices.json
+```
+
+```json
+{
+	"total": 1,
+	"devices": [
+		{
+			"hostname": "cpy-951032",
+			"instance_name": "Adafruit Feather ESP32-S2 TFT",
+			"port": 80,
+			"ip": "192.168.1.235"
+		}
+	]
+}
+```
+
+#### `/cp/serial/`
+
+
+Serves a basic serial terminal program when a `GET` request is received without the
+`Upgrade: websocket` header. Otherwise the socket is upgraded to a WebSocket. See WebSockets below for more detail.
+
+This is an authenticated endpoint in both modes.
+
 #### `/cp/version.json`
 
 Returns information about the device.
@@ -323,36 +361,6 @@ curl -v -L http://circuitpython.local/cp/version.json
 }
 ```
 
-#### `/cp/devices.json`
-
-Returns information about other devices found on the network using MDNS.
-
-* `total`: Total MDNS response count. May be more than in `devices` if internal limits were hit.
-* `devices`: List of discovered devices.
-	* `hostname`: MDNS hostname
-	* `instance_name`: MDNS instance name. Defaults to human readable board name.
-	* `port`: Port of CircuitPython Web API
-	* `ip`: IP address
-
-Example:
-```sh
-curl -v -L http://circuitpython.local/cp/devices.json
-```
-
-```json
-{
-	"total": 1,
-	"devices": [
-		{
-			"hostname": "cpy-951032",
-			"instance_name": "Adafruit Feather ESP32-S2 TFT",
-			"port": 80,
-			"ip": "192.168.1.235"
-		}
-	]
-}
-```
-
 ### Static files
 
 * `/favicon.ico` - Blinka
@@ -361,4 +369,12 @@ curl -v -L http://circuitpython.local/cp/devices.json
 
 ### WebSocket
 
-Coming soon!
+The CircuitPython serial interactions are available over a WebSocket. A WebSocket begins as a
+special HTTP request that gets upgraded to a WebSocket. Authentication happens before upgrading.
+
+WebSockets are *not* bare sockets once upgraded. Instead they have their own framing format for data.
+CircuitPython can handle PING and CLOSE opcodes. All others are treated as TEXT. Data to
+CircuitPython is expected to be masked UTF-8, as the spec requires. Data from CircuitPython to the
+client is unmasked. It is also unbuffered so the client will get a variety of frame sizes.
+
+Only one WebSocket at a time is supported.
diff --git a/supervisor/shared/web_workflow/static/welcome.html b/supervisor/shared/web_workflow/static/welcome.html
index 6ef1ccd767..139e9eba39 100644
--- a/supervisor/shared/web_workflow/static/welcome.html
+++ b/supervisor/shared/web_workflow/static/welcome.html
@@ -3,11 +3,12 @@
 
     CircuitPython
     
+    
     
 
 
     
 Welcome!

-    Welcome to CircuitPython's Web API. Go to the file browser to work with files in the CIRCUITPY drive. Make sure you've set CIRCUITPY_WEB_API_PASSWORD='somepassword' in /.env. Provide the password when the browser prompts for it. Leave the username blank.
+    Welcome to CircuitPython's Web API. Go to the file browser to work with files in the CIRCUITPY drive. Go to the serial terminal to see code output and interact with the REPL. Make sure you've set CIRCUITPY_WEB_API_PASSWORD='somepassword' in /.env. Provide the password when the browser prompts for it. Leave the username blank.
     
Device Info

     Board: 

     Version: 

diff --git a/supervisor/shared/web_workflow/web_workflow.c b/supervisor/shared/web_workflow/web_workflow.c
index c7bf673fdc..3f9920d1c7 100644
--- a/supervisor/shared/web_workflow/web_workflow.c
+++ b/supervisor/shared/web_workflow/web_workflow.c
@@ -1019,14 +1019,13 @@ static bool _reply(socketpool_socket_obj_t *socket, _request *request) {
         } else if (strcmp(path, "/version.json") == 0) {
             _reply_with_version_json(socket, request);
         } else if (strcmp(path, "/serial/") == 0) {
-            if (false && !request->authenticated) {
+            if (!request->authenticated) {
                 if (_api_password[0] != '\0') {
                     _reply_unauthorized(socket, request);
                 } else {
                     _reply_forbidden(socket, request);
                 }
             } else if (request->websocket) {
-                ESP_LOGI(TAG, "websocket!");
                 _reply_websocket_upgrade(socket, request);
             } else {
                 _REPLY_STATIC(socket, request, serial_html);
@@ -1059,7 +1058,6 @@ static bool _reply(socketpool_socket_obj_t *socket, _request *request) {
 }
 
 static void _reset_request(_request *request) {
-    ESP_LOGI(TAG, "reset request");
     request->state = STATE_METHOD;
     request->origin[0] = '\0';
     request->content_length = 0;
diff --git a/supervisor/shared/web_workflow/websocket.c b/supervisor/shared/web_workflow/websocket.c
index 7a2f37b168..8d3941b434 100644
--- a/supervisor/shared/web_workflow/websocket.c
+++ b/supervisor/shared/web_workflow/websocket.c
@@ -48,7 +48,6 @@ void websocket_init(void) {
 }
 
 void websocket_handoff(socketpool_socket_obj_t *socket) {
-    ESP_LOGI(TAG, "socket handed off");
     cp_serial.socket = *socket;
     cp_serial.closed = false;
     cp_serial.opcode = 0;
@@ -57,7 +56,6 @@ void websocket_handoff(socketpool_socket_obj_t *socket) {
     // Mark the original socket object as closed without telling the lower level.
     socket->connected = false;
     socket->num = -1;
-    ESP_LOGI(TAG, "socket hand off done");
 }
 
 bool websocket_connected(void) {
@@ -90,7 +88,6 @@ static void _read_next_frame_header(void) {
     if (cp_serial.frame_index == 0 && _read_byte(&h)) {
         cp_serial.frame_index++;
         cp_serial.opcode = h & 0xf;
-        ESP_LOGI(TAG, "fin %d opcode %x", h >> 7, cp_serial.opcode);
     }
     if (cp_serial.frame_index == 1 && _read_byte(&h)) {
         cp_serial.frame_index++;
@@ -108,8 +105,6 @@ static void _read_next_frame_header(void) {
         if (cp_serial.masked) {
             cp_serial.frame_len += 4;
         }
-
-        ESP_LOGI(TAG, "mask %d length %x", cp_serial.masked, len);
     }
     while (cp_serial.frame_index >= 2 &&
            cp_serial.frame_index < (cp_serial.payload_len_size + 2) &&
@@ -133,7 +128,6 @@ static void _read_next_frame_header(void) {
         if (cp_serial.frame_index == cp_serial.frame_len) {
             uint8_t opcode = 0x8; // CLOSE
             if (cp_serial.opcode == 0x9) {
-                ESP_LOGI(TAG, "websocket ping");
                 opcode = 0xA; // PONG
             } else {
                 // Set the TCP socket to send immediately so that we send the payload back before
@@ -160,7 +154,6 @@ static void _read_next_frame_header(void) {
         if (cp_serial.payload_remaining == 0) {
             cp_serial.frame_index = 0;
             if (cp_serial.opcode == 0x8) {
-                ESP_LOGI(TAG, "websocket closed");
                 cp_serial.closed = true;
 
                 common_hal_socketpool_socket_close(&cp_serial.socket);
@@ -199,7 +192,6 @@ bool websocket_available(void) {
 char websocket_read_char(void) {
     uint8_t c;
     _read_next_payload_byte(&c);
-    ESP_LOGI(TAG, "read %c", c);
     return c;
 }
 
@@ -232,7 +224,6 @@ static void _websocket_send(_websocket *ws, const char *text, size_t len) {
     char copy[len];
     memcpy(copy, text, len);
     copy[len] = '\0';
-    ESP_LOGI(TAG, "sent over websocket: %s", copy);
 }
 
 void websocket_write(const char *text, size_t len) {

From 425a0efeca5da563f981bc8dd271bf7f9421fb5d Mon Sep 17 00:00:00 2001
From: Scott Shawcroft 
Date: Mon, 11 Jul 2022 14:53:20 -0700
Subject: [PATCH 07/10] A bit more cleanup

---
 shared-bindings/hashlib/Hash.c                |  7 -------
 .../shared/web_workflow/static/serial.js      |  2 --
 supervisor/shared/web_workflow/web_workflow.c | 21 ++++++++-----------
 3 files changed, 9 insertions(+), 21 deletions(-)

diff --git a/shared-bindings/hashlib/Hash.c b/shared-bindings/hashlib/Hash.c
index 5dab05fa10..e27b71ef78 100644
--- a/shared-bindings/hashlib/Hash.c
+++ b/shared-bindings/hashlib/Hash.c
@@ -26,13 +26,6 @@
 
 #include "shared-bindings/hashlib/Hash.h"
 
-// #include "shared-bindings/util.h"
-
-// #include "shared/runtime/buffer_helper.h"
-// #include "shared/runtime/interrupt_char.h"
-
-// #include "py/mperrno.h"
-// #include "py/mphal.h"
 #include "py/obj.h"
 #include "py/objproperty.h"
 #include "py/objstr.h"
diff --git a/supervisor/shared/web_workflow/static/serial.js b/supervisor/shared/web_workflow/static/serial.js
index 595bbe6653..6ecc2b1659 100644
--- a/supervisor/shared/web_workflow/static/serial.js
+++ b/supervisor/shared/web_workflow/static/serial.js
@@ -21,7 +21,6 @@ function onSubmit() {
     input.focus();
 }
 
-// Connect to Web Socket
 ws = new WebSocket("ws://" + window.location.host + "/cp/serial/");
 
 ws.onopen = function() {
@@ -32,7 +31,6 @@ var setting_title = false;
 var encoder = new TextEncoder();
 var left_count = 0;
 ws.onmessage = function(e) {
-  // e.data contains received string.
   if (e.data == "\x1b]0;") {
     setting_title = true;
     title.textContent = "";
diff --git a/supervisor/shared/web_workflow/web_workflow.c b/supervisor/shared/web_workflow/web_workflow.c
index 3f9920d1c7..6cf765b8f9 100644
--- a/supervisor/shared/web_workflow/web_workflow.c
+++ b/supervisor/shared/web_workflow/web_workflow.c
@@ -522,10 +522,16 @@ static void _reply_redirect(socketpool_socket_obj_t *socket, _request *request,
         "HTTP/1.1 301 Moved Permanently\r\n",
         "Connection: close\r\n",
         "Content-Length: 0\r\n",
-        "Location: http://", hostname, ".local", path, "\r\n", NULL);
+        "Location: ", NULL);
+    if (request->websocket) {
+        _send_str(socket, "ws");
+    } else {
+        _send_str(socket, "http");
+    }
+
+    _send_strs(socket, "://", hostname, ".local", path, "\r\n", NULL);
     _cors_header(socket, request);
     _send_str(socket, "\r\n");
-    ESP_LOGI(TAG, "redirect");
 }
 
 static void _reply_directory_json(socketpool_socket_obj_t *socket, _request *request, FF_DIR *dir, const char *request_path, const char *path) {
@@ -853,10 +859,7 @@ static void _reply_static(socketpool_socket_obj_t *socket, _request *request, co
 
 #define _REPLY_STATIC(socket, request, filename) _reply_static(socket, request, filename, filename##_length, filename##_content_type)
 
-
-
 static void _reply_websocket_upgrade(socketpool_socket_obj_t *socket, _request *request) {
-    ESP_LOGI(TAG, "websocket!");
     // Compute accept key
     hashlib_hash_obj_t hash;
     common_hal_hashlib_new(&hash, "sha1");
@@ -876,8 +879,6 @@ static void _reply_websocket_upgrade(socketpool_socket_obj_t *socket, _request *
         "Sec-WebSocket-Accept: ", encoded_accept, "\r\n",
         "\r\n", NULL);
     websocket_handoff(socket);
-
-    ESP_LOGI(TAG, "socket upgrade done");
     // socket is now closed and "disconnected".
 }
 
@@ -885,7 +886,7 @@ static bool _reply(socketpool_socket_obj_t *socket, _request *request) {
     if (request->redirect) {
         _reply_redirect(socket, request, request->path);
     } else if (strlen(request->origin) > 0 && !_origin_ok(request->origin)) {
-        ESP_LOGI(TAG, "bad origin %s", request->origin);
+        ESP_LOGE(TAG, "bad origin %s", request->origin);
         _reply_forbidden(socket, request);
     } else if (memcmp(request->path, "/fs/", 4) == 0) {
         if (!request->authenticated) {
@@ -1199,7 +1200,6 @@ static void _process_request(socketpool_socket_obj_t *socket, _request *request)
         return;
     }
     bool reload = _reply(socket, request);
-    ESP_LOGI(TAG, "reply done");
     _reset_request(request);
     autoreload_resume(AUTORELOAD_SUSPEND_WEB);
     if (reload) {
@@ -1217,12 +1217,10 @@ void supervisor_web_workflow_background(void) {
         uint32_t port;
         int newsoc = socketpool_socket_accept(&listening, (uint8_t *)&ip, &port);
         if (newsoc == -EBADF) {
-            ESP_LOGI(TAG, "listen closed");
             common_hal_socketpool_socket_close(&listening);
             return;
         }
         if (newsoc > 0) {
-            ESP_LOGI(TAG, "new socket %d", newsoc);
             // Close the active socket because we have another we accepted.
             if (!common_hal_socketpool_socket_get_closed(&active)) {
                 common_hal_socketpool_socket_close(&active);
@@ -1243,7 +1241,6 @@ void supervisor_web_workflow_background(void) {
 
     // If we have a request in progress, continue working on it.
     if (common_hal_socketpool_socket_get_connected(&active)) {
-        // ESP_LOGI(TAG, "active connected %d", active_request.in_progress);
         _process_request(&active, &active_request);
     }
 }

From 15fe3864574d0a92f8d9612f0ab1da976ffc3ee0 Mon Sep 17 00:00:00 2001
From: Scott Shawcroft 
Date: Tue, 12 Jul 2022 11:13:17 -0700
Subject: [PATCH 08/10] Fix build and minify html and js

---
 docs/library/hashlib.rst                   | 1 +
 requirements-dev.txt                       | 4 ++++
 supervisor/shared/web_workflow/websocket.c | 3 +++
 tools/gen_web_workflow_static.py           | 6 ++++++
 4 files changed, 14 insertions(+)

diff --git a/docs/library/hashlib.rst b/docs/library/hashlib.rst
index 8e5ebc2d1a..061f9fd1e0 100644
--- a/docs/library/hashlib.rst
+++ b/docs/library/hashlib.rst
@@ -5,6 +5,7 @@
 
 .. module:: hashlib
    :synopsis: hashing algorithms
+   :noindex:
 
 |see_cpython_module| :mod:`cpython:hashlib`.
 
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 0b2e08163a..3b4411bd3a 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -26,3 +26,7 @@ pyelftools
 
 # for stubs and annotations
 adafruit-circuitpython-typing
+
+# for web workflow minify
+minify_html
+jsmin
diff --git a/supervisor/shared/web_workflow/websocket.c b/supervisor/shared/web_workflow/websocket.c
index 8d3941b434..313e18a86d 100644
--- a/supervisor/shared/web_workflow/websocket.c
+++ b/supervisor/shared/web_workflow/websocket.c
@@ -26,6 +26,9 @@
 
 #include "supervisor/shared/web_workflow/websocket.h"
 
+// TODO: Remove ESP specific stuff. For now, it is useful as we refine the server.
+#include "esp_log.h"
+
 typedef struct {
     socketpool_socket_obj_t socket;
     uint8_t opcode;
diff --git a/tools/gen_web_workflow_static.py b/tools/gen_web_workflow_static.py
index 5f0febd084..b8c5baf619 100644
--- a/tools/gen_web_workflow_static.py
+++ b/tools/gen_web_workflow_static.py
@@ -4,6 +4,8 @@
 
 import argparse
 import gzip
+import minify_html
+import jsmin
 import mimetypes
 import pathlib
 
@@ -24,6 +26,10 @@ for f in args.files:
     variable = path.name.replace(".", "_")
     uncompressed = f.read()
     ulen = len(uncompressed)
+    if f.name.endswith(".html"):
+        uncompressed = minify_html.minify(uncompressed.decode("utf-8")).encode("utf-8")
+    elif f.name.endswith(".js"):
+        uncompressed = jsmin.jsmin(uncompressed.decode("utf-8")).encode("utf-8")
     compressed = gzip.compress(uncompressed)
     clen = len(compressed)
     compressed = ", ".join([hex(x) for x in compressed])

From 8093f8e55521cccaaf273bb5f821e40f65432608 Mon Sep 17 00:00:00 2001
From: Scott Shawcroft 
Date: Tue, 12 Jul 2022 14:12:25 -0700
Subject: [PATCH 09/10] Default gifio to camera setting

---
 py/circuitpy_mpconfig.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/py/circuitpy_mpconfig.mk b/py/circuitpy_mpconfig.mk
index de88dcc727..5d6ef0ddf6 100644
--- a/py/circuitpy_mpconfig.mk
+++ b/py/circuitpy_mpconfig.mk
@@ -233,7 +233,7 @@ CIRCUITPY_GETPASS ?= $(CIRCUITPY_FULL_BUILD)
 CFLAGS += -DCIRCUITPY_GETPASS=$(CIRCUITPY_GETPASS)
 
 ifeq ($(CIRCUITPY_DISPLAYIO),1)
-CIRCUITPY_GIFIO ?= $(CIRCUITPY_FULL_BUILD)
+CIRCUITPY_GIFIO ?= $(CIRCUITPY_CAMERA)
 else
 CIRCUITPY_GIFIO ?= 0
 endif

From 031c124a81d6129f9bf8fa47cea1066788d481ba Mon Sep 17 00:00:00 2001
From: Scott Shawcroft 
Date: Tue, 12 Jul 2022 14:12:39 -0700
Subject: [PATCH 10/10] Tweak serial page to work better in Chrome

---
 supervisor/shared/web_workflow/static/serial.html | 8 ++++----
 supervisor/shared/web_workflow/static/serial.js   | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/supervisor/shared/web_workflow/static/serial.html b/supervisor/shared/web_workflow/static/serial.html
index aaf416b8b3..0c13248904 100644
--- a/supervisor/shared/web_workflow/static/serial.html
+++ b/supervisor/shared/web_workflow/static/serial.html
@@ -1,15 +1,15 @@
 
 
-
 
   Simple client
-
+  
+  
   
 
 
-  

+  

     

-     
+    
   

   

     

diff --git a/supervisor/shared/web_workflow/static/serial.js b/supervisor/shared/web_workflow/static/serial.js
index 6ecc2b1659..86ec077e92 100644
--- a/supervisor/shared/web_workflow/static/serial.js
+++ b/supervisor/shared/web_workflow/static/serial.js
@@ -57,7 +57,7 @@ ws.onerror = function(e) {
   set_enabled(false);
 };
 
-input.onbeforeinput = function(e) {
+input.addEventListener("beforeinput", function(e) {
   if (e.inputType == "insertLineBreak") {
     ws.send("\r");
     input.value = "";
@@ -68,7 +68,7 @@ input.onbeforeinput = function(e) {
   } else if (e.inputType == "deleteContentBackward") {
     ws.send("\b");
   }
-}
+});
 
 let ctrl_c = document.querySelector("#c");
 ctrl_c.onclick = function() {