Merge branch 'micropython:master' into master

Author: wemos

CONTRIBUTING.md

@@ -102,15 +102,20 @@ Pages](https://docs.github.com/en/pages):
    "truthy" value).
 5. The settings for GitHub Actions and GitHub Pages features should not need to
    be changed from the repository defaults, unless you've explicitly disabled
-   them.
+   Actions or Pages in your fork.
 
 The next time you push commits to a branch in your fork, GitHub Actions will run
 an additional step in the "Build All Packages" workflow named "Publish Packages
-for branch".
+for branch". This step runs in *your fork*, but if you open a pull request then
+this workflow is not shown in the Pull Request's "Checks". These run in the
+upstream repository. Navigate to your fork's Actions tab in order to see
+the additional "Publish Packages for branch" step.
 
 Anyone can then install these packages as described under [Installing packages
-from forks](README.md#installing-packages-from-forks). The exact commands are also
-quoted in the GitHub Actions log for the "Publish Packages for branch" step.
+from forks](README.md#installing-packages-from-forks).
+
+The exact command is also quoted in the GitHub Actions log in your fork's
+Actions for the "Publish Packages for branch" step of "Build All Packages".
 
 #### Opting Back Out
 

micropython/bluetooth/aioble-central/manifest.py

@@ -1,4 +1,4 @@
-metadata(version="0.2.2")
+metadata(version="0.3.0")
 
 require("aioble-core")
 

micropython/bluetooth/aioble-core/manifest.py

@@ -1,4 +1,4 @@
-metadata(version="0.3.0")
+metadata(version="0.4.0")
 
 package(
     "aioble",

micropython/bluetooth/aioble/aioble/central.py

@@ -104,7 +104,9 @@ async def _cancel_pending():
 
 # Start connecting to a peripheral.
 # Call device.connect() rather than using method directly.
-async def _connect(connection, timeout_ms):
+async def _connect(
+    connection, timeout_ms, scan_duration_ms, min_conn_interval_us, max_conn_interval_us
+):
     device = connection.device
     if device in _connecting:
         return
@@ -122,7 +124,13 @@ async def _connect(connection, timeout_ms):
 
     try:
         with DeviceTimeout(None, timeout_ms):
-            ble.gap_connect(device.addr_type, device.addr)
+            ble.gap_connect(
+                device.addr_type,
+                device.addr,
+                scan_duration_ms,
+                min_conn_interval_us,
+                max_conn_interval_us,
+            )
 
             # Wait for the connected IRQ.
             await connection._event.wait()

micropython/bluetooth/aioble/aioble/device.py

@@ -132,14 +132,26 @@ def __str__(self):
     def addr_hex(self):
         return binascii.hexlify(self.addr, ":").decode()
 
-    async def connect(self, timeout_ms=10000):
+    async def connect(
+        self,
+        timeout_ms=10000,
+        scan_duration_ms=None,
+        min_conn_interval_us=None,
+        max_conn_interval_us=None,
+    ):
         if self._connection:
             return self._connection
 
         # Forward to implementation in central.py.
         from .central import _connect
 
-        await _connect(DeviceConnection(self), timeout_ms)
+        await _connect(
+            DeviceConnection(self),
+            timeout_ms,
+            scan_duration_ms,
+            min_conn_interval_us,
+            max_conn_interval_us,
+        )
 
         # Start the device task that will clean up after disconnection.
         self._connection._run_task()

micropython/bluetooth/aioble/manifest.py

@@ -3,7 +3,7 @@
 # code. This allows (for development purposes) all the files to live in the
 # one directory.
 
-metadata(version="0.5.2")
+metadata(version="0.6.0")
 
 # Default installation gives you everything. Install the individual
 # components (or a combination of them) if you want a more minimal install.

micropython/lora/lora-sx126x/lora/sx126x.py

@@ -152,7 +152,7 @@ def __init__(
             dio3_tcxo_start_time_us if dio3_tcxo_millivolts else 0
         )
 
-        self._buf = bytearray(9)  # shared buffer for commands
+        self._buf_view = memoryview(bytearray(9))  # shared buffer for commands
 
         # These settings are kept in the object (as can't read them back from the modem)
         self._output_power = 14
@@ -704,11 +704,11 @@ def _cmd(self, fmt, *write_args, n_read=0, write_buf=None, read_buf=None):
         # have happened well before _cmd() is called again.
         self._wait_not_busy(self._busy_timeout)
 
-        # Pack write_args into _buf and wrap a memoryview of the correct length around it
+        # Pack write_args into slice of _buf_view memoryview of correct length
         wrlen = struct.calcsize(fmt)
-        assert n_read + wrlen <= len(self._buf)  # if this fails, make _buf bigger!
-        struct.pack_into(fmt, self._buf, 0, *write_args)
-        buf = memoryview(self._buf)[: (wrlen + n_read)]
+        assert n_read + wrlen <= len(self._buf_view)  # if this fails, make _buf bigger!
+        struct.pack_into(fmt, self._buf_view, 0, *write_args)
+        buf = self._buf_view[: (wrlen + n_read)]
 
         if _DEBUG:
             print(">>> {}".format(buf[:wrlen].hex()))
@@ -723,7 +723,7 @@ def _cmd(self, fmt, *write_args, n_read=0, write_buf=None, read_buf=None):
         self._cs(1)
 
         if n_read > 0:
-            res = memoryview(buf)[wrlen : (wrlen + n_read)]  # noqa: E203
+            res = self._buf_view[wrlen : (wrlen + n_read)]  # noqa: E203
             if _DEBUG:
                 print("<<< {}".format(res.hex()))
             return res

micropython/lora/lora-sx126x/manifest.py

@@ -1,3 +1,3 @@
-metadata(version="0.1.2")
+metadata(version="0.1.3")
 require("lora")
 package("lora")

micropython/lora/lora-sx127x/lora/sx127x.py

@@ -416,13 +416,13 @@ def configure(self, lora_cfg):
             modem_config1 |= (self._coding_rate - 4) << _MODEM_CONFIG1_CODING_RATE_SHIFT
             update_mask |= _MODEM_CONFIG1_CODING_RATE_MASK << _MODEM_CONFIG1_CODING_RATE_SHIFT
 
-        self._reg_update(_REG_MODEM_CONFIG1, update_mask, modem_config1)
-
         if "implicit_header" in lora_cfg:
             self._implicit_header = lora_cfg["implicit_header"]
             modem_config1 |= _flag(_MODEM_CONFIG1_IMPLICIT_HEADER_MODE_ON, self._implicit_header)
             update_mask |= _MODEM_CONFIG1_IMPLICIT_HEADER_MODE_ON
 
+        self._reg_update(_REG_MODEM_CONFIG1, update_mask, modem_config1)
+
         # Update MODEM_CONFIG2, for any fields that changed
         modem_config2 = 0
         update_mask = 0

micropython/lora/lora-sx127x/manifest.py

@@ -1,3 +1,3 @@
-metadata(version="0.1.1")
+metadata(version="0.1.2")
 require("lora")
 package("lora")

micropython/umqtt.robust/example_sub_robust.py

@@ -19,8 +19,7 @@ def sub_cb(topic, msg):
 #
 # There can be a problem when a session for a given client exists,
 # but doesn't have subscriptions a particular application expects.
-# In this case, a session needs to be cleaned first. See
-# example_reset_session.py for an obvious way how to do that.
+# In this case, a session needs to be cleaned first.
 #
 # In an actual application, it's up to its developer how to
 # manage these issues. One extreme is to have external "provisioning"

micropython/usb/README.md

@@ -134,3 +134,15 @@ USB MIDI devices in MicroPython.
 
 The example [midi_example.py](examples/device/midi_example.py) demonstrates how
 to create a simple MIDI device to send MIDI data to and from the USB host.
+
+### Limitations
+
+#### Buffer thread safety
+
+The internal Buffer class that's used by most of the USB device classes expects data
+to be written to it (i.e. sent to the host) by only one thread. Bytes may be
+lost from the USB transfers if more than one thread (or a thread and a callback)
+try to write to the buffer simultaneously.
+
+If writing USB data from multiple sources, your code may need to add
+synchronisation (i.e. locks).

micropython/usb/usb-device/manifest.py

@@ -1,2 +1,2 @@
-metadata(version="0.1.0")
+metadata(version="0.2.0")
 package("usb")

micropython/usb/usb-device/usb/device/core.py

@@ -8,6 +8,14 @@
 import machine
 import struct
 
+try:
+    from _thread import get_ident
+except ImportError:
+
+    def get_ident():
+        return 0  # Placeholder, for no threading support
+
+
 _EP_IN_FLAG = const(1 << 7)
 
 # USB descriptor types
@@ -76,6 +84,8 @@ def __init__(self):
         self._itfs = {}  # Mapping from interface number to interface object, set by init()
         self._eps = {}  # Mapping from endpoint address to interface object, set by _open_cb()
         self._ep_cbs = {}  # Mapping from endpoint address to Optional[xfer callback]
+        self._cb_thread = None  # Thread currently running endpoint callback
+        self._cb_ep = None  # Endpoint number currently running callback
         self._usbd = machine.USBDevice()  # low-level API
 
     def init(self, *itfs, **kwargs):
@@ -100,6 +110,7 @@ def config(  # noqa: PLR0913
         device_protocol=0,
         config_str=None,
         max_power_ma=None,
+        remote_wakeup=False,
     ):
         # Configure the USB device with a set of interfaces, and optionally reconfiguring the
         # device and configuration descriptor fields
@@ -189,7 +200,7 @@ def maybe_set(value, idx):
         bmAttributes = (
             (1 << 7)  # Reserved
             | (0 if max_power_ma else (1 << 6))  # Self-Powered
-            # Remote Wakeup not currently supported
+            | ((1 << 5) if remote_wakeup else 0)
         )
 
         # Configuration string is optional but supported
@@ -242,8 +253,8 @@ def active(self, *optional_value):
         return self._usbd.active(*optional_value)
 
     def _open_itf_cb(self, desc):
-        # Singleton callback from TinyUSB custom class driver, when USB host does
-        # Set Configuration. Called once per interface or IAD.
+        # Callback from TinyUSB lower layer, when USB host does Set
+        # Configuration. Called once per interface or IAD.
 
         # Note that even if the configuration descriptor contains an IAD, 'desc'
         # starts from the first interface descriptor in the IAD and not the IAD
@@ -281,7 +292,7 @@ def _open_itf_cb(self, desc):
             itf.on_open()
 
     def _reset_cb(self):
-        # Callback when the USB device is reset by the host
+        # TinyUSB lower layer callback when the USB device is reset by the host
 
         # Allow interfaces to respond to the reset
         for itf in self._itfs.values():
@@ -292,13 +303,13 @@ def _reset_cb(self):
         self._ep_cbs = {}
 
     def _submit_xfer(self, ep_addr, data, done_cb=None):
-        # Singleton function to submit a USB transfer (of any type except control).
+        # Submit a USB transfer (of any type except control) to TinyUSB lower layer.
         #
         # Generally, drivers should call Interface.submit_xfer() instead. See
         # that function for documentation about the possible parameter values.
         if ep_addr not in self._eps:
             raise ValueError("ep_addr")
-        if self._ep_cbs[ep_addr]:
+        if self._xfer_pending(ep_addr):
             raise RuntimeError("xfer_pending")
 
         # USBDevice callback may be called immediately, before Python execution
@@ -308,15 +319,32 @@ def _submit_xfer(self, ep_addr, data, done_cb=None):
         self._ep_cbs[ep_addr] = done_cb or True
         return self._usbd.submit_xfer(ep_addr, data)
 
+    def _xfer_pending(self, ep_addr):
+        # Returns True if a transfer is pending on this endpoint.
+        #
+        # Generally, drivers should call Interface.xfer_pending() instead. See that
+        # function for more documentation.
+        return self._ep_cbs[ep_addr] or (self._cb_ep == ep_addr and self._cb_thread != get_ident())
+
     def _xfer_cb(self, ep_addr, result, xferred_bytes):
-        # Singleton callback from TinyUSB custom class driver when a transfer completes.
+        # Callback from TinyUSB lower layer when a transfer completes.
         cb = self._ep_cbs.get(ep_addr, None)
+        self._cb_thread = get_ident()
+        self._cb_ep = ep_addr  # Track while callback is running
         self._ep_cbs[ep_addr] = None
-        if callable(cb):
-            cb(ep_addr, result, xferred_bytes)
+
+        # In most cases, 'cb' is a callback function for the transfer. Can also be:
+        # - True (for a transfer with no callback)
+        # - None (TinyUSB callback arrived for invalid endpoint, or no transfer.
+        #   Generally unlikely, but may happen in transient states.)
+        try:
+            if callable(cb):
+                cb(ep_addr, result, xferred_bytes)
+        finally:
+            self._cb_ep = None
 
     def _control_xfer_cb(self, stage, request):
-        # Singleton callback from TinyUSB custom class driver when a control
+        # Callback from TinyUSB lower layer when a control
         # transfer is in progress.
         #
         # stage determines appropriate responses (possible values
@@ -528,7 +556,12 @@ def xfer_pending(self, ep_addr):
         # Return True if a transfer is already pending on ep_addr.
         #
         # Only one transfer can be submitted at a time.
-        return _dev and bool(_dev._ep_cbs[ep_addr])
+        #
+        # The transfer is marked pending while a completion callback is running
+        # for that endpoint, unless this function is called from the callback
+        # itself. This makes it simple to submit a new transfer from the
+        # completion callback.
+        return _dev and _dev._xfer_pending(ep_addr)
 
     def submit_xfer(self, ep_addr, data, done_cb=None):
         # Submit a USB transfer (of any type except control)

python-stdlib/base64/manifest.py

@@ -1,6 +1,5 @@
-metadata(version="3.3.5")
+metadata(version="3.3.6")
 
 require("binascii")
-require("struct")
 
 module("base64.py")

tools/build.py

@@ -64,7 +64,7 @@
 
 # index.json is:
 # {
-#   "v": 1,                      <-- file format version
+#   "v": 2,                      <-- file format version
 #   "updated": <utc-seconds-since-1970>,
 #   "packages": {
 #     {
@@ -78,7 +78,9 @@
 #         "7": ["0.2", "0.3", "0.4"],
 #         ...                    <-- Other bytecode versions
 #         "py": ["0.1", "0.2", "0.3", "0.4"]
-#       }
+#       },
+#       // The following entries were added in file format version 2.
+#       path: "micropython/bluetooth/aioble",
 #     },
 #     ...
 #   }
@@ -122,7 +124,7 @@
 import time
 
 
-_JSON_VERSION_INDEX = 1
+_JSON_VERSION_INDEX = 2
 _JSON_VERSION_PACKAGE = 1
 
 
@@ -268,7 +270,7 @@ def _copy_as_py(
 
 # Update to the latest metadata, and add any new versions to the package in
 # the index json.
-def _update_index_package_metadata(index_package_json, metadata, mpy_version):
+def _update_index_package_metadata(index_package_json, metadata, mpy_version, package_path):
     index_package_json["version"] = metadata.version or ""
     index_package_json["author"] = ""  # TODO: Make manifestfile.py capture this.
     index_package_json["description"] = metadata.description or ""
@@ -283,6 +285,9 @@ def _update_index_package_metadata(index_package_json, metadata, mpy_version):
                 print("  New version {}={}".format(v, metadata.version))
                 index_package_json["versions"][v].append(metadata.version)
 
+    # The following entries were added in file format version 2.
+    index_package_json["path"] = package_path
+
 
 def build(output_path, hash_prefix_len, mpy_cross_path):
     import manifestfile
@@ -318,7 +323,8 @@ def build(output_path, hash_prefix_len, mpy_cross_path):
 
     for lib_dir in lib_dirs:
         for manifest_path in glob.glob(os.path.join(lib_dir, "**", "manifest.py"), recursive=True):
-            print("{}".format(os.path.dirname(manifest_path)))
+            package_path = os.path.dirname(manifest_path)
+            print("{}".format(package_path))
             # .../foo/manifest.py -> foo
             package_name = os.path.basename(os.path.dirname(manifest_path))
 
@@ -342,7 +348,9 @@ def build(output_path, hash_prefix_len, mpy_cross_path):
                 }
                 index_json["packages"].append(index_package_json)
 
-            _update_index_package_metadata(index_package_json, manifest.metadata(), mpy_version)
+            _update_index_package_metadata(
+                index_package_json, manifest.metadata(), mpy_version, package_path
+            )
 
             # This is the package json that mip/mpremote downloads.
             mpy_package_json = {