DonkeyCar Framework
Setup, Training & Autonomous Driving

# Refresh package lists — safe even if Pi OS was recently updated
sudo apt-get update --allow-releaseinfo-change

# Upgrade all installed packages to the latest available versions
sudo apt-get upgrade

sudo raspi-config

# Verify the I2C kernel device file exists
ls /dev/i2c-*
# Expected: /dev/i2c-1

# Install i2c-tools, then scan for connected I2C devices
sudo apt install -y i2c-tools
i2cdetect -y 1
# Each cell shows -- if no device is at that address, or a hex address if a device is found

# Verify DonkeyCar installed correctly and the car directory was created
python3 -c "import donkeycar; print('DonkeyCar version:', donkeycar.__version__)"
ls ~/mycar
# Expected output includes: manage.py  myconfig.py  models/  data/

pip install depthai==2.21.2.0

# Confirm the camera is detected at the USB level before testing the driver
lsusb
# Look for a line containing: Intel Corp. Movidius MyriadX   or the USB ID   03e7:2485

# Write the udev rule file (copy and run this entire block as one command)
sudo tee /etc/udev/rules.d/80-movidius.rules > /dev/null <<'EOF'
# Intel Movidius / Luxonis OAK-D-Lite (MyriadX) — allow non-root user access
SUBSYSTEM=="usb", ATTR{idVendor}=="03e7", MODE:="0666"
EOF

# Apply the new rule immediately — no reboot required for this step
sudo udevadm control --reload-rules
sudo udevadm trigger

sudo usermod -aG plugdev $USER
sudo usermod -aG dialout $USER
sudo reboot

python3 - << 'EOF'
import depthai as dai
print("DepthAI version:", dai.__version__)

# List all cameras visible on USB (should show OAK-D-LITE)
print("Available devices:", dai.Device.getAllAvailableDevices())

# Open the camera and read its identity
with dai.Device() as device:
    print("Opened device:", device.getDeviceName())
    print("MXID:", device.getMxId())
    print("USB speed:", device.getUsbSpeed())
EOF

DepthAI version: 2.21.2.0
Available devices: [DeviceInfo(OAK-D-LITE-FF)]
Opened device: OAK-D-LITE-FF
MXID: 18443010D10BF40F00
USB speed: UsbSpeed.HIGH

ls /dev/ttyACM*
# Expected: /dev/ttyACM0   (may appear as ttyACM1 if the Oak-D camera is also connected)

# If nothing appears, check the kernel log for USB enumeration errors
dmesg | grep -i "usb\|tty\|acm" | tail -20

pip install git+https://github.com/LiamBindle/PyVESC.git@master

# Verify both pyvesc and pyserial installed correctly
python3 -c "import pyvesc; import serial; print('pyvesc OK, pyserial', serial.__version__)"
# Expected: pyvesc OK, pyserial 3.5

# Confirm your user account is a member of the dialout group
groups $USER | grep dialout
# Expected output contains: dialout

# Confirm the device file is owned by the dialout group
ls -la /dev/ttyACM0
# Expected: crw-rw---- 1 root dialout 166, 0 ...

sudo apt install -y joystick evtest

sudo evtest
# Select the event number for "Logitech Gamepad F710" from the list
# Move the analog sticks — you should see lines like:
#   Event: type 3 (EV_ABS), code 0 (ABS_X), value 14253
# Press Ctrl+C to exit when done

python3 - << 'EOF'
import os
js_exists = os.path.exists("/dev/input/js0")
print("Joystick device /dev/input/js0 present:", js_exists)
assert js_exists, "ERROR: /dev/input/js0 not found — is the dongle plugged in and the controller powered on?"
print("Joystick ready.")
EOF
# Expected:
# Joystick device /dev/input/js0 present: True
# Joystick ready.

python3 - << 'EOF'
import tensorflow as tf
print("TensorFlow version:", tf.__version__)
EOF
# Expected: TensorFlow version: 2.15.1

python3 - << 'EOF'
import tensorflow as tf
a = tf.constant([[1.0, 2.0],[3.0, 4.0]])
b = tf.constant([[5.0, 6.0],[7.0, 8.0]])
c = tf.matmul(a, b)
print("matmul result:\n", c.numpy())
EOF

matmul result:
 [[19. 22.]
 [43. 50.]]

python3 - << 'EOF'
import tensorflow as tf
print("TF version:", tf.__version__)
print("Built with CUDA:", tf.test.is_built_with_cuda())
print("Physical devices:", tf.config.list_physical_devices())
EOF

TF version: 2.15.1
Built with CUDA: False
Physical devices: [PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU')]

# Find the exact path to VESC.py in your virtualenv
find ~/env -name "VESC.py" -path "*/pyvesc/*"
# Expected: /home/pi/env/lib/python3.11/site-packages/pyvesc/VESC/VESC.py

nano ~/env/lib/python3.11/site-packages/pyvesc/VESC/VESC.py
# Once inside nano: press Ctrl+W, type "get_firmware_version", press Enter

# check firmware version and set GetValue fields to old values if pre version 3.xx
version = self.get_firmware_version()
if int(version.split('.')[0]) < 3:
    GetValues.fields = pre_v3_33_fields

# check firmware version and set GetValue fields to old values if pre version 3.xx
version = self.get_firmware_version()
# PATCH: Safely handle None or unexpected version strings from modern VESC firmware
if not version or str(version).lower() == "none":
    version = "3.0"    # Default to v3 behaviour if the version string cannot be read
if int(str(version).split('.')[0]) < 3:
    GetValues.fields = pre_v3_33_fields

python3 -c "import pyvesc; import serial; print('pyvesc OK, pyserial', serial.__version__)"
# Expected: pyvesc OK, pyserial 3.5

nano ~/env/lib/python3.11/site-packages/donkeycar/parts/tub_v2.py
# Once inside nano: press Ctrl+W, type "_image_file_name", press Enter

name = Tub._image_file_name(self.manifest.current_index, key, ext='.png')

name = Tub._image_file_name(self.manifest.current_index, key, extension='.png')

cd ~/mycar
python3 manage.py drive    # Drive for ~5 seconds using the joystick, then press Ctrl+C

# Inspect the tub that was just created — you should see multiple numbered .png files
ls data/tub_*/images/ | head -5
# Expected: files named like:
#   0_cam-image_array_.png
#   1_cam-image_array_.png
#   2_cam-image_array_.png
# If you see only ONE file with no .png extension, the patch was not saved correctly.

nano ~/env/lib/python3.11/site-packages/donkeycar/parts/oak_d.py
# Once inside nano: press Ctrl+W, type "enable_rgb or self.enable_depth", press Enter

if self.enable_rgb or self.enable_depth:

    self.depth_queue: DataOutputQueue = self.oak_d_device.getOutputQueue(
        name="depth", maxSize=1, blocking=False
    )
    self.rgb_queue: DataOutputQueue = self.oak_d_device.getOutputQueue(
        "rgb", maxSize=1, blocking=False
    )

    depth_frame = self.get_frame(self.depth_queue)
    rgb_frame = self.get_frame(self.rgb_queue)

    self.depth_image = depth_frame
    self.color_image = rgb_frame

# if self.enable_rgb or self.enable_depth:
#
#     self.depth_queue: DataOutputQueue = self.oak_d_device.getOutputQueue(
#         name="depth", maxSize=1, blocking=False
#     )
#     self.rgb_queue: DataOutputQueue = self.oak_d_device.getOutputQueue(
#         "rgb", maxSize=1, blocking=False
#     )
#
#     depth_frame = self.get_frame(self.depth_queue)
#     rgb_frame = self.get_frame(self.rgb_queue)
#
#     self.depth_image = depth_frame
#     self.color_image = rgb_frame

# PATCH: Initialize RGB and depth queues independently based on myconfig.py settings.
# This prevents a RuntimeError when only one stream is enabled.
if self.enable_rgb:
    self.rgb_queue: DataOutputQueue = self.oak_d_device.getOutputQueue(
        "rgb", maxSize=1, blocking=False
    )
    rgb_frame = self.get_frame(self.rgb_queue)
    self.color_image = rgb_frame
else:
    self.color_image = None   # Explicitly set None to prevent AttributeError later

if self.enable_depth:
    self.depth_queue: DataOutputQueue = self.oak_d_device.getOutputQueue(
        name="depth", maxSize=1, blocking=False
    )
    depth_frame = self.get_frame(self.depth_queue)
    self.depth_image = depth_frame
else:
    self.depth_image = None   # Explicitly set None to prevent AttributeError later

python3 - << 'EOF'
import sys
sys.path.insert(0, '/home/pi/env/lib/python3.11/site-packages')
from donkeycar.parts.oak_d import OakD

# Test RGB-only mode — this is the standard lane-following configuration
cam = OakD(enable_rgb=True, enable_depth=False)
print("OakD RGB-only init: OK")
cam.shutdown()
EOF
# Expected: OakD RGB-only init: OK

nano ~/mycar/myconfig.py

# ── Image Transformations ────────────────────────────────────────────
TRANSFORMATIONS = ['RESIZE']
RESIZE_WIDTH = 160      # Small resolution keeps CPU inference fast on the Pi
RESIZE_HEIGHT = 120

# ── Drive Loop Timing ────────────────────────────────────────────────
DRIVE_LOOP_HZ = 20      # 20 Hz = one update every 50 ms; fast enough for smooth driving
MAX_LOOPS = None        # None = run until Ctrl+C; set a number to auto-stop after N loops

# ── Camera ───────────────────────────────────────────────────────────
CAMERA_TYPE = "OAKD"   # Use the Oak-D Lite via DepthAI
IMAGE_W = 160
IMAGE_H = 120
IMAGE_DEPTH = 3         # 3 = RGB color; set to 1 for grayscale
CAMERA_FRAMERATE = DRIVE_LOOP_HZ
CAMERA_VFLIP = False    # Set True if the camera image appears upside down
CAMERA_HFLIP = False    # Set True if the camera image appears mirrored left-right
CAMERA_INDEX = 0

# ── Drivetrain ───────────────────────────────────────────────────────
DRIVE_TRAIN_TYPE = "VESC"
VESC_MAX_SPEED_PERCENT = .5     # Start at 50% of max speed; increase gradually as the model improves
VESC_SERIAL_PORT = "/dev/ttyACM0"  # Change to /dev/ttyACM1 if the camera is using ACM0
VESC_HAS_SENSOR = True          # Hall effect encoder sensor is present on this drivetrain
VESC_START_HEARTBEAT = True     # Sends periodic keepalive packets to prevent VESC timeout
VESC_BAUDRATE = 115200
VESC_TIMEOUT = 0.05
VESC_STEERING_SCALE = 0.5   # Maps joystick range (-1 to 1) → VESC range (-0.5 to 0.5)
VESC_STEERING_OFFSET = 0.5  # Shifts the result so the final range is 0 to 1 (what VESC expects)

# ── Joystick ─────────────────────────────────────────────────────────
USE_JOYSTICK_AS_DEFAULT = True      # Removes the need to pass --js on every launch
JOYSTICK_MAX_THROTTLE = 0.5         # Limit top speed during data collection for safety
JOYSTICK_STEERING_SCALE = 1.0       # Set to -1.0 to reverse the steering direction
AUTO_RECORD_ON_THROTTLE = True      # Data records automatically while moving; stops when throttle is zero
CONTROLLER_TYPE = 'F710'            # ← CHANGE THIS to match your controller: ps3 | ps4 | xbox | F710
USE_NETWORKED_JS = False
NETWORK_JS_SERVER_IP = None
JOYSTICK_DEADZONE = 0.01            # Ignore stick inputs below 1% to eliminate drift at rest
JOYSTICK_THROTTLE_DIR = -1.0        # -1.0 flips the throttle axis so "up" on the stick = forward
JOYSTICK_DEVICE_FILE = "/dev/input/js0"

# ── Oak-D Specific ───────────────────────────────────────────────────
OAKD_RGB = True         # Capture RGB images for lane-following training
OAKD_DEPTH = False      # Depth stream not needed for basic lane following
OAKD_ID = None          # None = auto-detect (works when only one camera is connected)

# ── FPV Web Stream ────────────────────────────────────────────────────
USE_FPV = False         # Set True to stream the camera to http://<hostname>.local:8887

cd ~/mycar
python3 manage.py drive

# List all tub directories (each represents one drive session)
ls ~/mycar/data/

# Count how many image frames are in the most recent tub
ls ~/mycar/data/tub_*/images/*.png | wc -l
# 20 laps at 20 Hz for roughly 2 minutes each ≈ 2,400 frames minimum

# Inspect the manifest to confirm steering/throttle values were recorded alongside the images
head -3 ~/mycar/data/tub_*/manifest.json

cd ~/mycar
python3 train.py --tub data/tub_1_26_26 --model models/MyFirstModel.h5

cd ~/mycar
python3 train.py --tub data --model models/MyModel_AllData.h5

Epoch  1/100  loss: 0.8432  val_loss: 0.7901   ← both decreasing: good
Epoch 10/100  loss: 0.2341  val_loss: 0.2189   ← converging toward the same value: good
Epoch 20/100  loss: 0.1102  val_loss: 0.1287   ← small gap between train and val: acceptable
Epoch 30/100  loss: 0.0423  val_loss: 0.2891   ← val_loss rising while train loss falls: overfitting

ls -lh ~/mycar/models/
# Expected: MyModel_AllData.h5   (file size is typically 2–8 MB)

# Quick sanity check — load the model and confirm its expected input and output shapes
python3 - << 'EOF'
import tensorflow as tf
model = tf.keras.models.load_model('/home/pi/mycar/models/MyModel_AllData.h5')
print("Model loaded OK. Input shape:", model.input_shape)
print("Model loaded OK. Output shape:", model.output_shape)
EOF
# Expected: Input shape: (None, 120, 160, 3)   Output shape: (None, 2)
# The 2 outputs are steering and throttle.

cd ~/mycar
python3 manage.py drive --model models/MyModel_AllData.h5