Merge pull request #2690 from adafruit/qualia_ornament

Adding code and UF2 for the qualia ornament

Qualia/Qualia_S3_OrnamentVideoPlayer/MjpegClass.h

@@ -0,0 +1,210 @@
+// SPDX-FileCopyrightText: 2023 Limor Fried for Adafruit Industries
+//
+// SPDX-License-Identifier: MIT
+
+/*******************************************************************************
+ * ESP32_JPEG Wrapper Class
+ *
+ * Dependent libraries:
+ * ESP32_JPEG: https://github.com/esp-arduino-libs/ESP32_JPEG.git
+ ******************************************************************************/
+#pragma once
+
+#if defined(ESP32)
+
+#define READ_BUFFER_SIZE 1024
+#define MAXOUTPUTSIZE (MAX_BUFFERED_PIXELS / 16 / 16)
+
+#include <FS.h>
+
+#include <ESP32_JPEG_Library.h>
+
+class MjpegClass
+{
+public:
+  bool setup(
+      Stream *input, uint8_t *mjpeg_buf,
+      uint16_t *output_buf, size_t output_buf_size, bool useBigEndian)
+  {
+    _input = input;
+    _mjpeg_buf = mjpeg_buf;
+    _output_buf = (uint8_t *)output_buf;
+    _output_buf_size = output_buf_size;
+    _useBigEndian = useBigEndian;
+    _inputindex = 0;
+
+    if (!_read_buf)
+    {
+      _read_buf = (uint8_t *)malloc(READ_BUFFER_SIZE);
+    }
+
+    if (!_read_buf)
+    {
+      return false;
+    }
+
+    return true;
+  }
+
+  bool readMjpegBuf()
+  {
+    if (_inputindex == 0)
+    {
+      _buf_read = _input->readBytes(_read_buf, READ_BUFFER_SIZE);
+      _inputindex += _buf_read;
+    }
+    _mjpeg_buf_offset = 0;
+    int i = 0;
+    bool found_FFD8 = false;
+    while ((_buf_read > 0) && (!found_FFD8))
+    {
+      i = 0;
+      while ((i < _buf_read) && (!found_FFD8))
+      {
+        if ((_read_buf[i] == 0xFF) && (_read_buf[i + 1] == 0xD8)) // JPEG header
+        {
+          // Serial.printf("Found FFD8 at: %d.\n", i);
+          found_FFD8 = true;
+        }
+        ++i;
+      }
+      if (found_FFD8)
+      {
+        --i;
+      }
+      else
+      {
+        _buf_read = _input->readBytes(_read_buf, READ_BUFFER_SIZE);
+      }
+    }
+    uint8_t *_p = _read_buf + i;
+    _buf_read -= i;
+    bool found_FFD9 = false;
+    if (_buf_read > 0)
+    {
+      i = 3;
+      while ((_buf_read > 0) && (!found_FFD9))
+      {
+        if ((_mjpeg_buf_offset > 0) && (_mjpeg_buf[_mjpeg_buf_offset - 1] == 0xFF) && (_p[0] == 0xD9)) // JPEG trailer
+        {
+          // Serial.printf("Found FFD9 at: %d.\n", i);
+          found_FFD9 = true;
+        }
+        else
+        {
+          while ((i < _buf_read) && (!found_FFD9))
+          {
+            if ((_p[i] == 0xFF) && (_p[i + 1] == 0xD9)) // JPEG trailer
+            {
+              found_FFD9 = true;
+              ++i;
+            }
+            ++i;
+          }
+        }
+
+        // Serial.printf("i: %d\n", i);
+        memcpy(_mjpeg_buf + _mjpeg_buf_offset, _p, i);
+        _mjpeg_buf_offset += i;
+        size_t o = _buf_read - i;
+        if (o > 0)
+        {
+          // Serial.printf("o: %d\n", o);
+          memcpy(_read_buf, _p + i, o);
+          _buf_read = _input->readBytes(_read_buf + o, READ_BUFFER_SIZE - o);
+          _p = _read_buf;
+          _inputindex += _buf_read;
+          _buf_read += o;
+          // Serial.printf("_buf_read: %d\n", _buf_read);
+        }
+        else
+        {
+          _buf_read = _input->readBytes(_read_buf, READ_BUFFER_SIZE);
+          _p = _read_buf;
+          _inputindex += _buf_read;
+        }
+        i = 0;
+      }
+      if (found_FFD9)
+      {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  bool decodeJpg()
+  {
+    _remain = _mjpeg_buf_offset;
+
+    // Generate default configuration
+    jpeg_dec_config_t config = {
+        .output_type = JPEG_RAW_TYPE_RGB565_BE,
+        .rotate = JPEG_ROTATE_0D,
+    };
+    // Create jpeg_dec
+    _jpeg_dec = jpeg_dec_open(&config);
+
+    // Create io_callback handle
+    _jpeg_io = (jpeg_dec_io_t *)calloc(1, sizeof(jpeg_dec_io_t));
+
+    // Create out_info handle
+    _out_info = (jpeg_dec_header_info_t *)calloc(1, sizeof(jpeg_dec_header_info_t));
+
+    // Set input buffer and buffer len to io_callback
+    _jpeg_io->inbuf = _mjpeg_buf;
+    _jpeg_io->inbuf_len = _remain;
+
+    jpeg_dec_parse_header(_jpeg_dec, _jpeg_io, _out_info);
+
+    _w = _out_info->width;
+    _h = _out_info->height;
+
+    if ((_w * _h * 2) > _output_buf_size)
+    {
+      return false;
+    }
+    _jpeg_io->outbuf = _output_buf;
+
+    jpeg_dec_process(_jpeg_dec, _jpeg_io);
+    jpeg_dec_close(_jpeg_dec);
+
+    free(_jpeg_io);
+    free(_out_info);
+
+    return true;
+  }
+
+  int16_t getWidth()
+  {
+    return _w;
+  }
+
+  int16_t getHeight()
+  {
+    return _h;
+  }
+
+private:
+  Stream *_input;
+  uint8_t *_mjpeg_buf;
+  uint8_t *_output_buf;
+  size_t _output_buf_size;
+  bool _useBigEndian;
+
+  uint8_t *_read_buf;
+  int32_t _mjpeg_buf_offset = 0;
+
+  jpeg_dec_handle_t *_jpeg_dec;
+  jpeg_dec_io_t *_jpeg_io;
+  jpeg_dec_header_info_t *_out_info;
+
+  int16_t _w = 0, _h = 0;
+
+  int32_t _inputindex = 0;
+  int32_t _buf_read;
+  int32_t _remain = 0;
+};
+
+#endif // defined(ESP32)

Qualia/Qualia_S3_OrnamentVideoPlayer/Qualia_S3_OrnamentVideoPlayer.ino

@@ -0,0 +1,244 @@
+// SPDX-FileCopyrightText: 2023 Limor Fried for Adafruit Industries
+//
+// SPDX-License-Identifier: MIT
+
+/*******************************************************************************
+ * Motion JPEG Image Viewer
+ * This is a simple Motion JPEG image viewer example
+
+encode with
+ffmpeg -i "wash.mp4" -vf "fps=10,vflip,hflip,scale=-1:480:flags=lanczos,crop=480:480" -pix_fmt yuvj420p -q:v 9 wash.mjpeg
+
+ ******************************************************************************/
+#define MJPEG_FOLDER       "/videos" // cannot be root!
+#define MJPEG_OUTPUT_SIZE  (480 * 480 * 2)      // memory for a output image frame
+#define MJPEG_BUFFER_SIZE (MJPEG_OUTPUT_SIZE / 5) // memory for a single JPEG frame
+#define MJPEG_LOOPS        0
+
+#include <Arduino_GFX_Library.h>
+#include <Adafruit_CST8XX.h>
+//#include <SD.h>      // uncomment either SD or SD_MMC
+#include <SD_MMC.h>
+
+Arduino_XCA9554SWSPI *expander = new Arduino_XCA9554SWSPI(
+    PCA_TFT_RESET, PCA_TFT_CS, PCA_TFT_SCK, PCA_TFT_MOSI,
+    &Wire, 0x3F);
+    
+Arduino_ESP32RGBPanel *rgbpanel = new Arduino_ESP32RGBPanel(
+    TFT_DE, TFT_VSYNC, TFT_HSYNC, TFT_PCLK,
+    TFT_R1, TFT_R2, TFT_R3, TFT_R4, TFT_R5,
+    TFT_G0, TFT_G1, TFT_G2, TFT_G3, TFT_G4, TFT_G5,
+    TFT_B1, TFT_B2, TFT_B3, TFT_B4, TFT_B5,
+    1 /* hsync_polarity */, 50 /* hsync_front_porch */, 2 /* hsync_pulse_width */, 44 /* hsync_back_porch */,
+    1 /* vsync_polarity */, 16 /* vsync_front_porch */, 2 /* vsync_pulse_width */, 18 /* vsync_back_porch */
+    //,1, 30000000
+    );
+
+Arduino_RGB_Display *gfx = new Arduino_RGB_Display(
+// 2.1" 480x480 round display
+    480 /* width */, 480 /* height */, rgbpanel, 0 /* rotation */, true /* auto_flush */,
+    expander, GFX_NOT_DEFINED /* RST */, TL021WVC02_init_operations, sizeof(TL021WVC02_init_operations));
+
+Adafruit_CST8XX ctp = Adafruit_CST8XX();  // This library also supports FT6336U!
+#define I2C_TOUCH_ADDR 0x15
+bool touchOK = false;
+
+#include <SD_MMC.h>
+
+#include "MjpegClass.h"
+static MjpegClass mjpeg;
+File mjpegFile, video_dir;
+uint8_t *mjpeg_buf;
+uint16_t *output_buf;
+
+unsigned long total_show_video = 0;
+
+void setup()
+{
+  Serial.begin(115200);
+  Serial.setDebugOutput(true);
+  //while(!Serial) delay(10);
+  Serial.println("MJPEG Video Playback Demo");
+
+#ifdef GFX_EXTRA_PRE_INIT
+  GFX_EXTRA_PRE_INIT();
+#endif
+
+  // Init Display
+  Wire.setClock(400000); // speed up I2C 
+  if (!gfx->begin()) {
+    Serial.println("gfx->begin() failed!");
+  }
+  gfx->fillScreen(BLUE);
+
+  expander->pinMode(PCA_TFT_BACKLIGHT, OUTPUT);
+  expander->digitalWrite(PCA_TFT_BACKLIGHT, HIGH);
+
+  //while (!SD.begin(ss, SPI, 64000000UL))
+  //SD_MMC.setPins(SCK /* CLK */, MOSI /* CMD/MOSI */, MISO /* D0/MISO */);
+  SD_MMC.setPins(SCK, MOSI /* CMD/MOSI */, MISO /* D0/MISO */, A0 /* D1 */, A1 /* D2 */, SS /* D3/CS */); // quad MMC!
+  while (!SD_MMC.begin("/root", true))
+  {
+    Serial.println(F("ERROR: File System Mount Failed!"));
+    gfx->println(F("ERROR: File System Mount Failed!"));
+    delay(1000);
+  }
+  Serial.println("Found SD Card");
+
+  //  open filesystem
+  //video_dir = SD.open(MJPEG_FOLDER);
+  video_dir = SD_MMC.open(MJPEG_FOLDER);
+  if (!video_dir || !video_dir.isDirectory()){
+     Serial.println("Failed to open " MJPEG_FOLDER " directory");
+     while (1) delay(100);
+  }
+  Serial.println("Opened Dir");
+
+  mjpeg_buf = (uint8_t *)malloc(MJPEG_BUFFER_SIZE);
+  if (!mjpeg_buf) {
+    Serial.println(F("mjpeg_buf malloc failed!"));
+    while (1) delay(100);
+  }
+  Serial.println("Allocated decoding buffer");
+
+  output_buf = (uint16_t *)heap_caps_aligned_alloc(16, MJPEG_OUTPUT_SIZE, MALLOC_CAP_8BIT);
+  if (!output_buf) {
+    Serial.println(F("output_buf malloc failed!"));
+    while (1) delay(100);
+  }
+
+  expander->pinMode(PCA_BUTTON_UP, INPUT);
+  expander->pinMode(PCA_BUTTON_DOWN, INPUT);
+
+  if (!ctp.begin(&Wire, I2C_TOUCH_ADDR)) {
+    Serial.println("No touchscreen found");
+    touchOK = false;
+  } else {
+    Serial.println("Touchscreen found");
+    touchOK = true;
+  }
+}
+
+void loop()
+{
+  /* variables */
+  int total_frames = 0;
+  unsigned long total_read_video = 0;
+  unsigned long total_decode_video = 0;
+  unsigned long start_ms, curr_ms;
+  uint8_t check_UI_count = 0;
+  int16_t x = -1, y = -1, w = -1, h = -1;
+  total_show_video = 0;
+
+  if (mjpegFile) mjpegFile.close();
+  Serial.println("looking for a file...");
+
+  if (!video_dir || !video_dir.isDirectory()){
+     Serial.println("Failed to open " MJPEG_FOLDER " directory");
+     while (1) delay(100);
+  }
+
+  // look for first mjpeg file
+  while ((mjpegFile = video_dir.openNextFile()) != 0) {
+    if (!mjpegFile.isDirectory()) {
+      Serial.print("  FILE: ");
+      Serial.print(mjpegFile.name());
+      Serial.print("  SIZE: ");
+      Serial.println(mjpegFile.size());
+      if ((strstr(mjpegFile.name(), ".mjpeg") != 0) || (strstr(mjpegFile.name(), ".MJPEG") != 0)) {
+        Serial.println("   <---- found a video!");
+        break;
+      }
+    }
+    if (mjpegFile) mjpegFile.close();
+  }
+
+  if (!mjpegFile || mjpegFile.isDirectory())
+  {
+    Serial.println(F("ERROR: Failed to find a MJPEG file for reading, resetting..."));
+    //gfx->println(F("ERROR: Failed to find a MJPEG file for reading"));
+
+    // We kept getting hard crashes when trying to rewindDirectory or close/open dir
+    // so we're just going to do a softreset
+    esp_sleep_enable_timer_wakeup(1000);
+    esp_deep_sleep_start(); 
+  }
+
+  bool done_looping = false;
+  while (!done_looping) {
+    mjpegFile.seek(0);
+    total_frames = 0;
+    total_read_video = 0;
+    total_decode_video = 0;
+    total_show_video = 0;
+
+    Serial.println(F("MJPEG start"));
+  
+    start_ms = millis();
+    curr_ms = millis();
+    if (! mjpeg.setup(&mjpegFile, mjpeg_buf, output_buf, MJPEG_OUTPUT_SIZE, true /* useBigEndian */)) {
+       Serial.println("mjpeg.setup() failed");
+       while (1) delay(100);
+    }
+  
+    while (mjpegFile.available() && mjpeg.readMjpegBuf())
+    {
+      // Read video
+      total_read_video += millis() - curr_ms;
+      curr_ms = millis();
+
+      // Play video
+      mjpeg.decodeJpg();
+      total_decode_video += millis() - curr_ms;
+      curr_ms = millis();
+
+      if (x == -1) {
+        w = mjpeg.getWidth();
+        h = mjpeg.getHeight();
+        x = (w > gfx->width()) ? 0 : ((gfx->width() - w) / 2);
+        y = (h > gfx->height()) ? 0 : ((gfx->height() - h) / 2);
+      }
+      gfx->draw16bitBeRGBBitmap(x, y, output_buf, w, h);
+      total_show_video += millis() - curr_ms;
+
+      curr_ms = millis();
+      total_frames++;
+      check_UI_count++;
+      if (check_UI_count >= 5) {
+        check_UI_count = 0;
+        Serial.print('.');
+        
+        if (! expander->digitalRead(PCA_BUTTON_DOWN)) {
+          Serial.println("\nDown pressed");
+          done_looping = true;
+          while (! expander->digitalRead(PCA_BUTTON_DOWN)) delay(10);
+          break;
+        }
+        if (! expander->digitalRead(PCA_BUTTON_UP)) {
+          Serial.println("\nUp pressed");
+          done_looping = true;
+          while (! expander->digitalRead(PCA_BUTTON_UP)) delay(10);
+          break;
+        }
+  
+        if (touchOK && ctp.touched()) {
+          CST_TS_Point p = ctp.getPoint(0);
+          Serial.printf("(%d, %d)\n", p.x, p.y);
+          done_looping = true;
+          break;
+        }
+      }
+    }
+    int time_used = millis() - start_ms;
+    Serial.println(F("MJPEG end"));
+    
+    float fps = 1000.0 * total_frames / time_used;
+    total_decode_video -= total_show_video;
+    Serial.printf("Total frames: %d\n", total_frames);
+    Serial.printf("Time used: %d ms\n", time_used);
+    Serial.printf("Average FPS: %0.1f\n", fps);
+    Serial.printf("Read MJPEG: %lu ms (%0.1f %%)\n", total_read_video, 100.0 * total_read_video / time_used);
+    Serial.printf("Decode video: %lu ms (%0.1f %%)\n", total_decode_video, 100.0 * total_decode_video / time_used);
+    Serial.printf("Show video: %lu ms (%0.1f %%)\n", total_show_video, 100.0 * total_show_video / time_used);
+  }
+}