Merge #716

716: Implement giant (chunked) DMA transfers r=Dirbaio a=matoushybl

There is a breaking change in the Channel trait in the method `remaining_transfers()`, this method however hasn't been used anywhere in the codebase and the return type changed from u16 to u32, so all of the previous use cases should work.

Co-authored-by: Matous Hybl <hyblmatous@gmail.com>

embassy-stm32/src/dcmi.rs

@@ -379,7 +379,23 @@ where
 
     /// This method starts the capture and finishes when both the dma transfer and DCMI finish the frame transfer.
     /// The implication is that the input buffer size must be exactly the size of the captured frame.
+    ///
+    /// Note that when `buffer.len() > 0xffff` the capture future requires some real-time guarantees to be upheld
+    /// (must be polled fast enough so the buffers get switched before data is overwritten).
+    /// It is therefore recommended that it is run on higher priority executor.
     pub async fn capture(&mut self, buffer: &mut [u32]) -> Result<(), Error> {
+        if buffer.len() <= 0xffff {
+            return self.capture_small(buffer).await;
+        } else {
+            #[cfg(feature = "unsafe-double-buffered-dma")]
+            return self.capture_giant(buffer).await;
+
+            #[cfg(not(feature = "unsafe-double-buffered-dma"))]
+            panic!("For DCMI transfers with length > 0xffff, the `unsafe-double-buffered-dma` must be enabled.");
+        }
+    }
+
+    async fn capture_small(&mut self, buffer: &mut [u32]) -> Result<(), Error> {
         let channel = &mut self.dma;
         let request = channel.request();
 
@@ -428,6 +444,122 @@ where
 
         result
     }
+
+    #[cfg(feature = "unsafe-double-buffered-dma")]
+    async fn capture_giant(&mut self, buffer: &mut [u32]) -> Result<(), Error> {
+        use crate::dma::TransferOptions;
+
+        let data_len = buffer.len();
+        let chunk_estimate = data_len / 0xffff;
+
+        let mut chunks = chunk_estimate + 1;
+        while data_len % chunks != 0 {
+            chunks += 1;
+        }
+
+        let chunk_size = data_len / chunks;
+
+        let mut remaining_chunks = chunks - 2;
+
+        let mut m0ar = buffer.as_mut_ptr();
+        let mut m1ar = unsafe { buffer.as_mut_ptr().add(chunk_size) };
+
+        let channel = &mut self.dma;
+        let request = channel.request();
+
+        let r = self.inner.regs();
+        let src = r.dr().ptr() as *mut u32;
+
+        unsafe {
+            channel.start_double_buffered_read(
+                request,
+                src,
+                m0ar,
+                m1ar,
+                chunk_size,
+                TransferOptions::default(),
+            );
+        }
+
+        let mut last_chunk_set_for_transfer = false;
+        let mut buffer0_last_accessible = false;
+        let dma_result = poll_fn(|cx| {
+            channel.set_waker(cx.waker());
+
+            let buffer0_currently_accessible = unsafe { channel.is_buffer0_accessible() };
+
+            // check if the accessible buffer changed since last poll
+            if buffer0_last_accessible == buffer0_currently_accessible {
+                return Poll::Pending;
+            }
+            buffer0_last_accessible = !buffer0_last_accessible;
+
+            if remaining_chunks != 0 {
+                if remaining_chunks % 2 == 0 && buffer0_currently_accessible {
+                    m0ar = unsafe { m0ar.add(2 * chunk_size) };
+                    unsafe { channel.set_buffer0(m0ar) }
+                    remaining_chunks -= 1;
+                } else if !buffer0_currently_accessible {
+                    m1ar = unsafe { m1ar.add(2 * chunk_size) };
+                    unsafe { channel.set_buffer1(m1ar) };
+                    remaining_chunks -= 1;
+                }
+            } else {
+                if buffer0_currently_accessible {
+                    unsafe { channel.set_buffer0(buffer.as_mut_ptr()) }
+                } else {
+                    unsafe { channel.set_buffer1(buffer.as_mut_ptr()) }
+                }
+                if last_chunk_set_for_transfer {
+                    channel.request_stop();
+                    return Poll::Ready(());
+                }
+                last_chunk_set_for_transfer = true;
+            }
+            Poll::Pending
+        });
+
+        Self::clear_interrupt_flags();
+        Self::enable_irqs();
+
+        let result = poll_fn(|cx| {
+            STATE.waker.register(cx.waker());
+
+            let ris = unsafe { crate::pac::DCMI.ris().read() };
+            if ris.err_ris() {
+                unsafe {
+                    crate::pac::DCMI.icr().write(|r| {
+                        r.set_err_isc(true);
+                    })
+                };
+                Poll::Ready(Err(Error::PeripheralError))
+            } else if ris.ovr_ris() {
+                unsafe {
+                    crate::pac::DCMI.icr().write(|r| {
+                        r.set_ovr_isc(true);
+                    })
+                };
+                Poll::Ready(Err(Error::Overrun))
+            } else if ris.frame_ris() {
+                unsafe {
+                    crate::pac::DCMI.icr().write(|r| {
+                        r.set_frame_isc(true);
+                    })
+                };
+                Poll::Ready(Ok(()))
+            } else {
+                Poll::Pending
+            }
+        });
+
+        unsafe { Self::toggle(true) };
+
+        let (_, result) = futures::future::join(dma_result, result).await;
+
+        unsafe { Self::toggle(false) };
+
+        result
+    }
 }
 
 mod sealed {

embassy-stm32/src/dma/bdma.rs

@@ -76,7 +76,6 @@ foreach_dma_channel! {
                 );
             }
 
-
             unsafe fn start_write_repeated<W: Word>(&mut self, _request: Request, repeated: W, count: usize, reg_addr: *mut W, options: TransferOptions) {
                 let buf = [repeated];
                 low_level_api::start_transfer(
@@ -119,6 +118,30 @@ foreach_dma_channel! {
                 );
             }
 
+            unsafe fn start_double_buffered_read<W: super::Word>(
+                &mut self,
+                _request: Request,
+                _reg_addr: *const W,
+                _buffer0: *mut W,
+                _buffer1: *mut W,
+                _buffer_len: usize,
+                _options: TransferOptions,
+            ) {
+                panic!("Unsafe double buffered mode is unavailable on BDMA");
+            }
+
+            unsafe fn set_buffer0<W: super::Word>(&mut self, _buffer: *mut W)  {
+                panic!("Unsafe double buffered mode is unavailable on BDMA");
+            }
+
+            unsafe fn set_buffer1<W: super::Word>(&mut self, _buffer: *mut W) {
+                panic!("Unsafe double buffered mode is unavailable on BDMA");
+            }
+
+            unsafe fn is_buffer0_accessible(&mut self) -> bool {
+                panic!("Unsafe double buffered mode is unavailable on BDMA");
+            }
+
             fn request_stop(&mut self){
                 unsafe {low_level_api::request_stop(pac::$dma_peri, $channel_num);}
             }
@@ -232,7 +255,7 @@ mod low_level_api {
         // get a handle on the channel itself
         let ch = dma.ch(ch as _);
         // read the remaining transfer count. If this is zero, the transfer completed fully.
-        ch.ndtr().read().ndt()
+        ch.ndtr().read().ndt() as u16
     }
 
     /// Sets the waker for the specified DMA channel

embassy-stm32/src/dma/dma.rs

@@ -41,15 +41,27 @@ impl From<FlowControl> for vals::Pfctrl {
     }
 }
 
+struct ChannelState {
+    waker: AtomicWaker,
+}
+
+impl ChannelState {
+    const fn new() -> Self {
+        Self {
+            waker: AtomicWaker::new(),
+        }
+    }
+}
+
 struct State {
-    ch_wakers: [AtomicWaker; DMA_CHANNEL_COUNT],
+    channels: [ChannelState; DMA_CHANNEL_COUNT],
 }
 
 impl State {
     const fn new() -> Self {
-        const AW: AtomicWaker = AtomicWaker::new();
+        const CH: ChannelState = ChannelState::new();
         Self {
-            ch_wakers: [AW; DMA_CHANNEL_COUNT],
+            channels: [CH; DMA_CHANNEL_COUNT],
         }
     }
 }
@@ -129,6 +141,46 @@ foreach_dma_channel! {
                 );
             }
 
+            unsafe fn start_double_buffered_read<W: Word>(
+                &mut self,
+                request: Request,
+                reg_addr: *const W,
+                buffer0: *mut W,
+                buffer1: *mut W,
+                buffer_len: usize,
+                options: TransferOptions,
+            ) {
+                low_level_api::start_dbm_transfer(
+                    pac::$dma_peri,
+                    $channel_num,
+                    request,
+                    vals::Dir::PERIPHERALTOMEMORY,
+                    reg_addr as *const u32,
+                    buffer0 as *mut u32,
+                    buffer1 as *mut u32,
+                    buffer_len,
+                    true,
+                    vals::Size::from(W::bits()),
+                    options,
+                    #[cfg(dmamux)]
+                    <Self as super::dmamux::sealed::MuxChannel>::DMAMUX_REGS,
+                    #[cfg(dmamux)]
+                    <Self as super::dmamux::sealed::MuxChannel>::DMAMUX_CH_NUM,
+                );
+            }
+
+            unsafe fn set_buffer0<W: Word>(&mut self, buffer: *mut W) {
+                low_level_api::set_dbm_buffer0(pac::$dma_peri, $channel_num, buffer as *mut u32);
+            }
+
+            unsafe fn set_buffer1<W: Word>(&mut self, buffer: *mut W) {
+                low_level_api::set_dbm_buffer1(pac::$dma_peri, $channel_num, buffer as *mut u32);
+            }
+
+            unsafe fn is_buffer0_accessible(&mut self) -> bool {
+                low_level_api::is_buffer0_accessible(pac::$dma_peri, $channel_num)
+            }
+
             fn request_stop(&mut self) {
                 unsafe {low_level_api::request_stop(pac::$dma_peri, $channel_num);}
             }
@@ -151,7 +203,6 @@ foreach_dma_channel! {
                 }
             }
         }
-
         impl crate::dma::Channel for crate::peripherals::$channel_peri { }
     };
 }
@@ -212,6 +263,94 @@ mod low_level_api {
         });
     }
 
+    pub unsafe fn start_dbm_transfer(
+        dma: pac::dma::Dma,
+        channel_number: u8,
+        request: Request,
+        dir: vals::Dir,
+        peri_addr: *const u32,
+        mem0_addr: *mut u32,
+        mem1_addr: *mut u32,
+        mem_len: usize,
+        incr_mem: bool,
+        data_size: vals::Size,
+        options: TransferOptions,
+        #[cfg(dmamux)] dmamux_regs: pac::dmamux::Dmamux,
+        #[cfg(dmamux)] dmamux_ch_num: u8,
+    ) {
+        #[cfg(dmamux)]
+        super::super::dmamux::configure_dmamux(dmamux_regs, dmamux_ch_num, request);
+
+        trace!(
+            "Starting DBM transfer with 0: 0x{:x}, 1: 0x{:x}, len: 0x{:x}",
+            mem0_addr as u32,
+            mem1_addr as u32,
+            mem_len
+        );
+
+        // "Preceding reads and writes cannot be moved past subsequent writes."
+        fence(Ordering::SeqCst);
+
+        reset_status(dma, channel_number);
+
+        let ch = dma.st(channel_number as _);
+        ch.par().write_value(peri_addr as u32);
+        ch.m0ar().write_value(mem0_addr as u32);
+        // configures the second buffer for DBM
+        ch.m1ar().write_value(mem1_addr as u32);
+        ch.ndtr().write_value(regs::Ndtr(mem_len as _));
+        ch.cr().write(|w| {
+            w.set_dir(dir);
+            w.set_msize(data_size);
+            w.set_psize(data_size);
+            w.set_pl(vals::Pl::VERYHIGH);
+            if incr_mem {
+                w.set_minc(vals::Inc::INCREMENTED);
+            } else {
+                w.set_minc(vals::Inc::FIXED);
+            }
+            w.set_pinc(vals::Inc::FIXED);
+            w.set_teie(true);
+            w.set_tcie(true);
+
+            #[cfg(dma_v1)]
+            w.set_trbuff(true);
+
+            #[cfg(dma_v2)]
+            w.set_chsel(request);
+
+            // enable double buffered mode
+            w.set_dbm(vals::Dbm::ENABLED);
+
+            w.set_pburst(options.pburst.into());
+            w.set_mburst(options.mburst.into());
+            w.set_pfctrl(options.flow_ctrl.into());
+
+            w.set_en(true);
+        });
+    }
+
+    pub unsafe fn set_dbm_buffer0(dma: pac::dma::Dma, channel_number: u8, mem_addr: *mut u32) {
+        // get a handle on the channel itself
+        let ch = dma.st(channel_number as _);
+        // change M0AR to the new address
+        ch.m0ar().write_value(mem_addr as _);
+    }
+
+    pub unsafe fn set_dbm_buffer1(dma: pac::dma::Dma, channel_number: u8, mem_addr: *mut u32) {
+        // get a handle on the channel itself
+        let ch = dma.st(channel_number as _);
+        // change M1AR to the new address
+        ch.m1ar().write_value(mem_addr as _);
+    }
+
+    pub unsafe fn is_buffer0_accessible(dma: pac::dma::Dma, channel_number: u8) -> bool {
+        // get a handle on the channel itself
+        let ch = dma.st(channel_number as _);
+        // check the current target register value
+        ch.cr().read().ct() == vals::Ct::MEMORY1
+    }
+
     /// Stops the DMA channel.
     pub unsafe fn request_stop(dma: pac::dma::Dma, channel_number: u8) {
         // get a handle on the channel itself
@@ -246,7 +385,7 @@ mod low_level_api {
 
     /// Sets the waker for the specified DMA channel
     pub unsafe fn set_waker(state_number: usize, waker: &Waker) {
-        STATE.ch_wakers[state_number].register(waker);
+        STATE.channels[state_number].waker.register(waker);
     }
 
     pub unsafe fn reset_status(dma: pac::dma::Dma, channel_number: u8) {
@@ -260,9 +399,9 @@ mod low_level_api {
     }
 
     /// Safety: Must be called with a matching set of parameters for a valid dma channel
-    pub unsafe fn on_irq_inner(dma: pac::dma::Dma, channel_num: u8, index: u8) {
+    pub unsafe fn on_irq_inner(dma: pac::dma::Dma, channel_num: u8, state_index: u8) {
         let channel_num = channel_num as usize;
-        let index = index as usize;
+        let state_index = state_index as usize;
 
         let cr = dma.st(channel_num).cr();
         let isr = dma.isr(channel_num / 4).read();
@@ -273,9 +412,16 @@ mod low_level_api {
                 dma.0 as u32, channel_num
             );
         }
+
         if isr.tcif(channel_num % 4) && cr.read().tcie() {
-            cr.write(|_| ()); // Disable channel interrupts with the default value.
-            STATE.ch_wakers[index].wake();
+            if cr.read().dbm() == vals::Dbm::DISABLED {
+                cr.write(|_| ()); // Disable channel with the default value.
+            } else {
+                // for double buffered mode, clear TCIF flag but do not stop the transfer
+                dma.ifcr(channel_num / 4)
+                    .write(|w| w.set_tcif(channel_num % 4, true));
+            }
+            STATE.channels[state_index].waker.wake();
         }
     }
 }

embassy-stm32/src/dma/mod.rs

@@ -76,6 +76,25 @@ pub(crate) mod sealed {
             options: TransferOptions,
         );
 
+        /// DMA double-buffered mode is unsafe as UB can happen when the hardware writes to a buffer currently owned by the software
+        /// more information can be found here: https://github.com/embassy-rs/embassy/issues/702
+        /// This feature is now used solely for the purposes of implementing giant DMA transfers required for DCMI
+        unsafe fn start_double_buffered_read<W: super::Word>(
+            &mut self,
+            request: Request,
+            reg_addr: *const W,
+            buffer0: *mut W,
+            buffer1: *mut W,
+            buffer_len: usize,
+            options: TransferOptions,
+        );
+
+        unsafe fn set_buffer0<W: super::Word>(&mut self, buffer: *mut W);
+
+        unsafe fn set_buffer1<W: super::Word>(&mut self, buffer: *mut W);
+
+        unsafe fn is_buffer0_accessible(&mut self) -> bool;
+
         /// Requests the channel to stop.
         /// NOTE: The channel does not immediately stop, you have to wait
         /// for `is_running() = false`.

examples/stm32h7/memory.x

@@ -1,5 +1,5 @@
 MEMORY
 {
     FLASH : ORIGIN = 0x8000000, LENGTH = 1024K
-    RAM : ORIGIN = 0x24000000, LENGTH = 128K
-}
+    RAM : ORIGIN = 0x24000000, LENGTH = 384K
+}