first tests

.gitignore

+*__pycache__
+*.vscode
\ No newline at end of file

2023-12_usb-real-data-rates.md

+## 2023 12 12 
+
+I've been trying to sort out how to get enough data out of the NIST printer to get to the "millisecond benchy" goal - part of that is figuring what real underlying data rates on USB serial links are. 
+
+To start, I spun up ... 
+
+![4byte](images/2023-12-12_ingest-histogram-single-source-pck-32.png)
+
+![64byte](images/2023-12-12_ingest-histogram-single-source-pck-64.png)
+
+![128byte](images/2023-12-12_ingest-histogram-single-source-pck-128.png)
+
+I would want to improve these by relating instantaneous real-data-rates with packet length and time deltas... i.e. for a 128 byte packet, supposing a 1ms interval, we have only `128k Byte / sec` whereas the spec looks for `296k Byte / sec` (for all the data). 
+
+So, not done at all:
+
+- usb packet length is actually 64 bytes, so zero-delimited would mean we should see a step size around ~ 60 bytes, not 64 as tested ? 
+- plot data-rates as well as per-packet delays
+- get COBS up to 512, or whatever the underlying USB packet size is ? how does that plot ?
+- what about multiple devices ?
+- what about flow control ?
+- how does it compare on a linux machine ? (the same machine, with linux) 
+  - how about a raspberry pi ? 
+
+I think the big point will be that we will want to measure these links in-situ, to do intelligent (feedback!) systems design. 
+
+## 2023 12 20 
+
+So - yeah, I want to keep working through this today, and I'm going to bundle code in here as well. 
+
+Firstly, it seems like USB 2.0 Full Speed has core frame sizes of 512 bytes, which we should endeavour to use - finishing our little diagrams above. That means this NanoCOBS implementation as well? 
+
+OK so I'm going to see if I can't swap in nanocobs, then test again at 32 byte packets;
+
+
+
+--- 
+
+... then test to see if we can (1) get some backpressure going on and (2) intelligently control data rates, maybe culminating in some demo of a high-throughput application where we can visualize data flows, and see i.e. how traffic patterns change over time (if new devices are added, for example). 
+
+--- 
+
+And then a next step would be to get the COBS thing up to doing flow-control, and testing TS as well, pushing a first-hello-world-for-each... if we can do that today, and show that we can RP2040 UART PIO, that's good shit. 
+
+---
+
+## Results
+
+The data printer has a spec for 1.4Mbits/s total data, which initial tests look to not be capable of (out of one USB port) (2023-12-12 images). 
\ No newline at end of file

code/serial_sink/cobs_usb_serial.py

+from cobs import cobs
+import serial
+
+
+class CobsUsbSerial:
+    def __init__(self, port, baudrate=115200):
+        self.port = port
+        self.ser = serial.Serial(port, baudrate=baudrate, timeout=1)
+
+    def write(self, data: bytes):
+        data_enc = cobs.encode(data) + b"\x00"
+        self.ser.write(data_enc)
+
+    def read(self):
+        data_enc = self.ser.read_until(b"\x00")
+        data = cobs.decode(data_enc[:-1])
+        return data

code/serial_sink/serial_list.py

+import serial.tools.list_ports
+
+def list_serial_ports():
+    ports = serial.tools.list_ports.comports()
+    for port in ports:
+        print(f"Port: {port.device}")
+        print(f" - Description: {port.description}")
+        if port.serial_number:
+            print(f" - Serial Number: {port.serial_number}")
+        if port.manufacturer:
+            print(f" - Manufacturer: {port.manufacturer}")
+        if port.product:
+            print(f" - Product: {port.product}")
+        if port.vid is not None:
+            print(f" - VID: {port.vid:04X}")
+        if port.pid is not None:
+            print(f" - PID: {port.pid:04X}")
+        print()
+
+list_serial_ports()

code/serial_sink/sink.py

+from cobs_usb_serial import CobsUsbSerial 
+import struct 
+import numpy as np 
+import pandas as pd 
+import matplotlib.pyplot as plt 
+
+ser = CobsUsbSerial("COM23") 
+
+stamp_count = 10000
+pck_len = 128
+
+stamps = np.zeros(stamp_count)
+
+for i in range(stamp_count):
+  bts = ser.read()
+  if len(bts) == pck_len:
+    stamp = struct.unpack('=I', bts[:4])
+    stamps[i] = stamp[0]
+
+print("stamps, ", stamps)
+
+df = pd.DataFrame({'timestamps': stamps})
+
+df['deltas'] = df['timestamps'].diff() 
+
+# clean NaN's 
+df = df.dropna()
+
+# wipe obviously-wrong deltas (i.e. the 1st, which goes 0-start-us) 
+df = df[df['deltas'] < 100000]
+
+# Bandwidth calculation (bytes/us to Mbits/s)
+# 1 byte/us = 8 Mbits/s
+df['bandwidth'] = (pck_len / df['deltas']) * 8 * 1e3  # Convert to Mbits/s
+
+# Plotting
+fig, ax1 = plt.subplots(figsize=(11, 5))
+
+# Primary x-axis (time deltas)
+df['deltas'].plot(kind='hist', bins=100, ax=ax1)
+ax1.set_xlabel('Time-Stamp Deltas (us)')
+ax1.set_ylabel(f'Frequency (of {stamp_count})')
+
+# Secondary x-axis (bandwidth)
+ax2 = ax1.twiny()
+ax2.set_xlabel('Estimated Bandwidth (Mbits/s)')
+
+# Set the limits of the secondary axis based on the primary axis
+new_tick_locations = np.linspace(df['deltas'].min(), df['deltas'].max(), num=len(ax1.get_xticks()))
+
+# Convert tick locations to bandwidth
+# ax2.set_xlim([pck_len / x * 8 * 1e3 for x in new_tick_locations])
+
+plt.title(f'Single-Source COBS Data Sink Deltas, pck_len={pck_len}')
+
+plt.tight_layout()
+
+plt.show()

code/serial_source/COBSUSBSerial.cpp

+// link ! 
+// TODO: pls use nanocobs, for encode- and decode-in-place, to save big (!) memory 
+// on new link layer... to fit into D11s... 
+
+#include "COBSUSBSerial.h"
+#include "cobs.h"
+
+
+#if defined(ARDUINO_ARCH_RP2040) || defined(ARDUINO_ARCH_RP2040)
+COBSUSBSerial::COBSUSBSerial(SerialUSB* _usbcdc){
+  usbcdc = _usbcdc;
+}
+#else 
+COBSUSBSerial::COBSUSBSerial(Serial_* _usbcdc){
+  usbcdc = _usbcdc;
+}
+#endif 
+
+void COBSUSBSerial::begin(void){
+  usbcdc->begin(9600);
+}
+
+void COBSUSBSerial::loop(void){
+  // check RX side:
+  // while data & not-full, 
+  while(usbcdc->available() && rxBufferLen == 0){
+    rxBuffer[rxBufferWp ++] = usbcdc->read();
+    if(rxBuffer[rxBufferWp - 1] == 0){
+      // decoding in place should always work: COBS doesn't revisit bytes 
+      // encoding in place would be a different trick, and would require the use of 
+      // nanocobs from this lad: https://github.com/charlesnicholson/nanocobs 
+      size_t len = cobsDecode(rxBuffer, 255, rxBuffer);
+      // now we are with-packet, set length and reset write pointer 
+      // len includes the trailing 0, rm that... 
+      rxBufferLen = len - 1; 
+      rxBufferWp = 0;
+    }
+  }
+
+  // check tx side, 
+  while(txBufferLen && usbcdc->availableForWrite()){
+    // ship a byte, 
+    usbcdc->write(txBuffer[txBufferRp ++]);
+    // if done, mark empty
+    if(txBufferRp >= txBufferLen){
+      txBufferLen = 0;
+      txBufferRp = 0;
+    }
+  }
+}
+
+size_t COBSUSBSerial::getPacket(uint8_t* dest){
+  if(rxBufferLen > 0){
+    memcpy(dest, rxBuffer, rxBufferLen);
+    size_t len = rxBufferLen;
+    rxBufferLen = 0;
+    return len;
+  } else {
+    return 0;
+  }
+}
+
+boolean COBSUSBSerial::clearToRead(void){
+  return (rxBufferLen > 0);
+}
+
+void COBSUSBSerial::send(uint8_t* packet, size_t len){
+  // ship it! blind! 
+  size_t encodedLen = cobsEncode(packet, len, txBuffer);
+  // stuff 0 byte, 
+  txBuffer[encodedLen] = 0;
+  txBufferLen = encodedLen + 1;
+  txBufferRp = 0;
+}
+
+boolean COBSUSBSerial::clearToSend(void){
+  // we're CTS if we have nothing in the outbuffer, 
+  return (txBufferLen == 0);
+}
+
+// we should do some... work with this, i.e. 
+// keepalives, to detect if other-end is open or not... 
+boolean COBSUSBSerial::isOpen(void){
+  if(usbcdc){
+    return true;
+  } else {
+    return false; 
+  }
+}
\ No newline at end of file

code/serial_source/COBSUSBSerial.h

+// example cobs-encoded usb-serial link 
+
+#include <Arduino.h>
+
+class COBSUSBSerial {
+  public: 
+    #if defined(ARDUINO_ARCH_RP2040) || defined(ARDUINO_ARCH_RP2040)
+    COBSUSBSerial(SerialUSB* _usbcdc);
+    #else 
+    COBSUSBSerial(Serial_* _usbcdc);
+    #endif 
+    void begin(void);
+    void loop(void);
+    // check & read,
+    boolean clearToRead(void);
+    size_t getPacket(uint8_t* dest);
+    // clear ahead?
+    boolean clearToSend(void);
+    // open at all?
+    boolean isOpen(void);
+    // transmit a packet of this length 
+    void send(uint8_t* packet, size_t len);
+  private: 
+    #if defined(ARDUINO_ARCH_RP2040) || defined(ARDUINO_ARCH_RP2040)
+    SerialUSB* usbcdc = nullptr;
+    #else 
+    Serial_* usbcdc = nullptr;
+    #endif 
+    // buffer, write pointer, length, 
+    uint8_t rxBuffer[255];
+    uint8_t rxBufferWp = 0;
+    uint8_t rxBufferLen = 0;
+    // ibid, 
+    uint8_t txBuffer[255];
+    uint8_t txBufferRp = 0;
+    uint8_t txBufferLen = 0;
+};
\ No newline at end of file

code/serial_source/cobs.cpp

+/*
+utils/cobs.cpp
+
+Jake Read at the Center for Bits and Atoms
+(c) Massachusetts Institute of Technology 2019
+
+This work may be reproduced, modified, distributed, performed, and
+displayed for any purpose, but must acknowledge the osap project.
+Copyright is retained and must be preserved. The work is provided as is;
+no warranty is provided, and users accept all liability.
+*/
+
+#include "cobs.h"
+// str8 crib from
+// https://en.wikipedia.org/wiki/Consistent_Overhead_Byte_Stuffing
+
+/** COBS encode data to buffer
+	@param data Pointer to input data to encode
+	@param length Number of bytes to encode
+	@param buffer Pointer to encoded output buffer
+	@return Encoded buffer length in bytes
+	@note doesn't write stop delimiter
+*/
+
+size_t cobsEncode(const void *data, size_t length, uint8_t *buffer){
+
+	uint8_t *encode = buffer; // Encoded byte pointer
+	uint8_t *codep = encode++; // Output code pointer
+	uint8_t code = 1; // Code value
+
+	for (const uint8_t *byte = (const uint8_t *)data; length--; ++byte){
+		if (*byte) // Byte not zero, write it
+			*encode++ = *byte, ++code;
+
+		if (!*byte || code == 0xff){ // Input is zero or block completed, restart
+			*codep = code, code = 1, codep = encode;
+			if (!*byte || length)
+				++encode;
+		}
+	}
+	*codep = code;  // Write final code value
+	return encode - buffer;
+}
+
+/** COBS decode data from buffer
+	@param buffer Pointer to encoded input bytes
+	@param length Number of bytes to decode
+	@param data Pointer to decoded output data
+	@return Number of bytes successfully decoded
+	@note Stops decoding if delimiter byte is found
+*/
+
+size_t cobsDecode(const uint8_t *buffer, size_t length, void *data){
+
+	const uint8_t *byte = buffer; // Encoded input byte pointer
+	uint8_t *decode = (uint8_t *)data; // Decoded output byte pointer
+
+	for (uint8_t code = 0xff, block = 0; byte < buffer + length; --block){
+		if (block) // Decode block byte
+			*decode++ = *byte++;
+		else
+		{
+			if (code != 0xff) // Encoded zero, write it
+				*decode++ = 0;
+			block = code = *byte++; // Next block length
+			if (code == 0x00) // Delimiter code found
+				break;
+		}
+	}
+
+	return decode - (uint8_t *)data;
+}

code/serial_source/cobs.h

+/*
+utils/cobs.h
+
+consistent overhead byte stuffing implementation
+
+Jake Read at the Center for Bits and Atoms
+(c) Massachusetts Institute of Technology 2019
+
+This work may be reproduced, modified, distributed, performed, and
+displayed for any purpose, but must acknowledge the osap project.
+Copyright is retained and must be preserved. The work is provided as is;
+no warranty is provided, and users accept all liability.
+*/
+
+#ifndef UTIL_COBS_H_
+#define UTIL_COBS_H_
+
+#include <Arduino.h>
+
+size_t cobsEncode(const void *data, size_t length, uint8_t *buffer);
+
+size_t cobsDecode(const uint8_t *buffer, size_t length, void *data);
+
+#endif

code/serial_source/serial_source.ino

+#include "COBSUSBSerial.h"
+
+// py-transport-tester 
+// using the RP2040 at 200MHz 
+
+#define MSG_GET_ID        7
+#define MSG_ECHO_TEST     11 
+#define MSG_SET_TARG_VEL  21
+#define MSG_SET_TARG_POS  22
+#define MSG_SET_POSITION  23 
+#define MSG_SET_CURRENT   24
+#define MSG_GET_STATES    25
+#define MSG_ACK           31 
+#define MSG_NACK          32
+
+COBSUSBSerial cobs(&Serial);
+
+void setup() {
+  cobs.begin();
+  pinMode(PIN_LED_R, OUTPUT);
+  pinMode(PIN_LED_G, OUTPUT);
+  pinMode(PIN_LED_B, OUTPUT);
+  digitalWrite(PIN_LED_R, HIGH);
+  digitalWrite(PIN_LED_G, HIGH);
+  digitalWrite(PIN_LED_B, HIGH);
+}
+
+union chunk_uint32 {
+  uint8_t bytes[4];
+  uint32_t u;
+};
+
+chunk_uint32 chunk;
+
+uint32_t lastBlink = 0;
+
+void loop() {
+  // run comms
+  cobs.loop();
+  // tx a stamp AFAP 
+  if(cobs.clearToSend()){
+    chunk.u = micros();
+    cobs.send(chunk.bytes, 128);
+    digitalWrite(PIN_LED_G, !digitalRead(PIN_LED_G));
+  }
+  // blink to see hangups 
+  if(lastBlink + 100 < millis()){
+    lastBlink = millis();
+    digitalWrite(PIN_LED_B, !digitalRead(PIN_LED_B));
+  }
+}