max/embassy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

rp/pio: handle all pio irqs in one handler

Browse Source 
dma does this too, also with 12 bits to check. this decreases code size
significantly (increasing speed when the cache is cold), frees up an
interrupt handler, and avoids read-modify-write cycles (which makes each
processed flag cheaper). due to more iterations per handler invocation
the actual runtime of the handler body remains roughly the
same (slightly faster at O2, slightly slower at Oz).

notably wakers are now kept in one large array indexed by the irq
register bit number instead of three different arrays, this allows for
machine code-level optimizations of waker lookups.
This commit is contained in:
pennae 2023-04-25 02:27:56 +02:00
parent ce04b732d1
commit a10850a6da
1 changed files with 44 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
embassy-rp/src/pio.rs
View File

@ -12,14 +12,29 @@ use crate::dma::{self, Channel, Transfer};
use crate::gpio::sealed::Pin as SealedPin; use crate::gpio::sealed::Pin as SealedPin;
use crate::gpio::{Drive, Pin, Pull, SlewRate}; use crate::gpio::{Drive, Pin, Pull, SlewRate};
use crate::pac::dma::vals::{DataSize, TreqSel}; use crate::pac::dma::vals::{DataSize, TreqSel};
use crate::{interrupt, pac, peripherals}; use crate::{interrupt, pac, peripherals, RegExt};
struct Wakers([AtomicWaker; 12]);
impl Wakers {
#[inline(always)]
fn fifo_in(&self) -> &[AtomicWaker] {
&self.0[0..4]
}
#[inline(always)]
fn fifo_out(&self) -> &[AtomicWaker] {
&self.0[4..8]
}
#[inline(always)]
fn irq(&self) -> &[AtomicWaker] {
&self.0[8..12]
}
}
const PIOS: [&pac::pio::Pio; 2] = [&pac::PIO0, &pac::PIO1]; const PIOS: [&pac::pio::Pio; 2] = [&pac::PIO0, &pac::PIO1];
const NEW_AW: AtomicWaker = AtomicWaker::new(); const NEW_AW: AtomicWaker = AtomicWaker::new();
const PIO_WAKERS_INIT: [AtomicWaker; 4] = [NEW_AW; 4]; const PIO_WAKERS_INIT: Wakers = Wakers([NEW_AW; 12]);
static FIFO_OUT_WAKERS: [[AtomicWaker; 4]; 2] = [PIO_WAKERS_INIT; 2]; static WAKERS: [Wakers; 2] = [PIO_WAKERS_INIT; 2];
static FIFO_IN_WAKERS: [[AtomicWaker; 4]; 2] = [PIO_WAKERS_INIT; 2];
static IRQ_WAKERS: [[AtomicWaker; 4]; 2] = [PIO_WAKERS_INIT; 2];
pub enum FifoJoin { pub enum FifoJoin {
/// Both TX and RX fifo is enabled /// Both TX and RX fifo is enabled
@ -40,83 +55,28 @@ const RXNEMPTY_MASK: u32 = 1 << 0;
const TXNFULL_MASK: u32 = 1 << 4; const TXNFULL_MASK: u32 = 1 << 4;
const SMIRQ_MASK: u32 = 1 << 8; const SMIRQ_MASK: u32 = 1 << 8;
#[interrupt]
unsafe fn PIO0_IRQ_1() {
use crate::pac;
let ints = pac::PIO0.irqs(1).ints().read().0;
let inte = pac::PIO0.irqs(1).inte();
for i in 0..4 {
// Check RXNEMPTY
if ints & (RXNEMPTY_MASK << i) != 0 {
inte.modify(|m| {
m.0 &= !(RXNEMPTY_MASK << i);
});
FIFO_IN_WAKERS[0][i].wake();
}
// Check IRQ flgs
if ints & (SMIRQ_MASK << i) != 0 {
inte.modify(|m| {
m.0 &= !(SMIRQ_MASK << i);
});
IRQ_WAKERS[0][i].wake();
}
}
}
#[interrupt]
unsafe fn PIO1_IRQ_1() {
use crate::pac;
let ints = pac::PIO1.irqs(1).ints().read().0;
let inte = pac::PIO1.irqs(1).inte();
for i in 0..4 {
// Check all RXNEMPTY
if ints & (RXNEMPTY_MASK << i) != 0 {
inte.modify(|m| {
m.0 &= !(RXNEMPTY_MASK << i);
});
FIFO_IN_WAKERS[1][i].wake();
}
// Check IRQ flgs
if ints & (SMIRQ_MASK << i) != 0 {
inte.modify(|m| {
m.0 &= !(SMIRQ_MASK << i);
});
IRQ_WAKERS[1][i].wake();
}
}
}
#[interrupt] #[interrupt]
unsafe fn PIO0_IRQ_0() { unsafe fn PIO0_IRQ_0() {
use crate::pac; use crate::pac;
let ints = pac::PIO0.irqs(0).ints().read().0; let ints = pac::PIO0.irqs(0).ints().read().0;
let inte = pac::PIO0.irqs(0).inte(); for bit in 0..12 {
//debug!("!{:04x}",ints); if ints & (1 << bit) != 0 {
// Check all TXNFULL WAKERS[0].0[bit].wake();
for i in 0..4 {
if ints & (TXNFULL_MASK << i) != 0 {
inte.modify(|m| {
m.0 &= !(TXNFULL_MASK << i);
});
FIFO_OUT_WAKERS[0][i].wake();
} }
} }
pac::PIO0.irqs(0).inte().write_clear(|m| m.0 = ints);
} }
#[interrupt] #[interrupt]
unsafe fn PIO1_IRQ_0() { unsafe fn PIO1_IRQ_0() {
use crate::pac; use crate::pac;
let ints = pac::PIO1.irqs(0).ints().read().0; let ints = pac::PIO1.irqs(0).ints().read().0;
let inte = pac::PIO1.irqs(0).inte(); for bit in 0..12 {
// Check all TXNFULL if ints & (1 << bit) != 0 {
for i in 0..4 { WAKERS[1].0[bit].wake();
if ints & (TXNFULL_MASK << i) != 0 {
inte.modify(|m| {
m.0 &= !(TXNFULL_MASK << i);
});
FIFO_OUT_WAKERS[1][i].wake();
} }
} }
pac::PIO1.irqs(0).inte().write_clear(|m| m.0 = ints);
} }
/// Future that waits for TX-FIFO to become writable /// Future that waits for TX-FIFO to become writable
@ -131,7 +91,7 @@ impl<'a, PIO: PioInstance, SM: PioStateMachine + Unpin> FifoOutFuture<'a, PIO, S
pub fn new(sm: &'a mut SM, value: u32) -> Self { pub fn new(sm: &'a mut SM, value: u32) -> Self {
unsafe { unsafe {
critical_section::with(|_| { critical_section::with(|_| {
let irq = PIO::IrqOut::steal(); let irq = PIO::Irq::steal();
irq.set_priority(interrupt::Priority::P3); irq.set_priority(interrupt::Priority::P3);
irq.enable(); irq.enable();
@ -153,9 +113,9 @@ impl<'d, PIO: PioInstance, SM: PioStateMachine + Unpin> Future for FifoOutFuture
if self.get_mut().sm.try_push_tx(value) { if self.get_mut().sm.try_push_tx(value) {
Poll::Ready(()) Poll::Ready(())
} else { } else {
FIFO_OUT_WAKERS[PIO::PIO_NO as usize][SM::Sm::SM_NO as usize].register(cx.waker()); WAKERS[PIO::PIO_NO as usize].fifo_out()[SM::Sm::SM_NO as usize].register(cx.waker());
unsafe { unsafe {
let irq = PIO::IrqOut::steal(); let irq = PIO::Irq::steal();
irq.disable(); irq.disable();
critical_section::with(|_| { critical_section::with(|_| {
PIOS[PIO::PIO_NO as usize].irqs(0).inte().modify(|m| { PIOS[PIO::PIO_NO as usize].irqs(0).inte().modify(|m| {
@ -193,7 +153,7 @@ impl<'a, PIO: PioInstance, SM: PioStateMachine> FifoInFuture<'a, PIO, SM> {
pub fn new(sm: &'a mut SM) -> Self { pub fn new(sm: &'a mut SM) -> Self {
unsafe { unsafe {
critical_section::with(|_| { critical_section::with(|_| {
let irq = PIO::IrqIn::steal(); let irq = PIO::Irq::steal();
irq.set_priority(interrupt::Priority::P3); irq.set_priority(interrupt::Priority::P3);
irq.enable(); irq.enable();
@ -213,12 +173,12 @@ impl<'d, PIO: PioInstance, SM: PioStateMachine> Future for FifoInFuture<'d, PIO,
if let Some(v) = self.sm.try_pull_rx() { if let Some(v) = self.sm.try_pull_rx() {
Poll::Ready(v) Poll::Ready(v)
} else { } else {
FIFO_IN_WAKERS[PIO::PIO_NO as usize][SM::Sm::SM_NO as usize].register(cx.waker()); WAKERS[PIO::PIO_NO as usize].fifo_in()[SM::Sm::SM_NO as usize].register(cx.waker());
unsafe { unsafe {
let irq = PIO::IrqIn::steal(); let irq = PIO::Irq::steal();
irq.disable(); irq.disable();
critical_section::with(|_| { critical_section::with(|_| {
PIOS[PIO::PIO_NO as usize].irqs(1).inte().modify(|m| { PIOS[PIO::PIO_NO as usize].irqs(0).inte().modify(|m| {
m.0 |= RXNEMPTY_MASK << SM::Sm::SM_NO; m.0 |= RXNEMPTY_MASK << SM::Sm::SM_NO;
}); });
}); });
@ -234,7 +194,7 @@ impl<'d, PIO: PioInstance, SM: PioStateMachine> Drop for FifoInFuture<'d, PIO, S
fn drop(&mut self) { fn drop(&mut self) {
unsafe { unsafe {
critical_section::with(|_| { critical_section::with(|_| {
PIOS[PIO::PIO_NO as usize].irqs(1).inte().modify(|m| { PIOS[PIO::PIO_NO as usize].irqs(0).inte().modify(|m| {
m.0 &= !(RXNEMPTY_MASK << SM::Sm::SM_NO); m.0 &= !(RXNEMPTY_MASK << SM::Sm::SM_NO);
}); });
}); });
@ -253,7 +213,7 @@ impl<'a, PIO: PioInstance> IrqFuture<PIO> {
pub fn new(irq_no: u8) -> Self { pub fn new(irq_no: u8) -> Self {
unsafe { unsafe {
critical_section::with(|_| { critical_section::with(|_| {
let irq = PIO::IrqSm::steal(); let irq = PIO::Irq::steal();
irq.set_priority(interrupt::Priority::P3); irq.set_priority(interrupt::Priority::P3);
irq.enable(); irq.enable();
@ -286,12 +246,12 @@ impl<'d, PIO: PioInstance> Future for IrqFuture<PIO> {
return Poll::Ready(()); return Poll::Ready(());
} }
IRQ_WAKERS[PIO::PIO_NO as usize][self.irq_no as usize].register(cx.waker()); WAKERS[PIO::PIO_NO as usize].irq()[self.irq_no as usize].register(cx.waker());
unsafe { unsafe {
let irq = PIO::IrqSm::steal(); let irq = PIO::Irq::steal();
irq.disable(); irq.disable();
critical_section::with(|_| { critical_section::with(|_| {
PIOS[PIO::PIO_NO as usize].irqs(1).inte().modify(|m| { PIOS[PIO::PIO_NO as usize].irqs(0).inte().modify(|m| {
m.0 |= SMIRQ_MASK << self.irq_no; m.0 |= SMIRQ_MASK << self.irq_no;
}); });
}); });
@ -305,7 +265,7 @@ impl<'d, PIO: PioInstance> Drop for IrqFuture<PIO> {
fn drop(&mut self) { fn drop(&mut self) {
unsafe { unsafe {
critical_section::with(|_| { critical_section::with(|_| {
PIOS[PIO::PIO_NO as usize].irqs(1).inte().modify(|m| { PIOS[PIO::PIO_NO as usize].irqs(0).inte().modify(|m| {
m.0 &= !(SMIRQ_MASK << self.irq_no); m.0 &= !(SMIRQ_MASK << self.irq_no);
}); });
}); });
@ -1223,23 +1183,17 @@ pub struct PioInstanceBase<const PIO_NO: u8> {}
pub trait PioInstance: Unpin { pub trait PioInstance: Unpin {
const PIO_NO: u8; const PIO_NO: u8;
type IrqOut: Interrupt; type Irq: Interrupt;
type IrqIn: Interrupt;
type IrqSm: Interrupt;
} }
impl PioInstance for PioInstanceBase<0> { impl PioInstance for PioInstanceBase<0> {
const PIO_NO: u8 = 0; const PIO_NO: u8 = 0;
type IrqOut = interrupt::PIO0_IRQ_0; type Irq = interrupt::PIO0_IRQ_0;
type IrqIn = interrupt::PIO0_IRQ_1;
type IrqSm = interrupt::PIO0_IRQ_1;
} }
impl PioInstance for PioInstanceBase<1> { impl PioInstance for PioInstanceBase<1> {
const PIO_NO: u8 = 1; const PIO_NO: u8 = 1;
type IrqOut = interrupt::PIO1_IRQ_0; type Irq = interrupt::PIO1_IRQ_0;
type IrqIn = interrupt::PIO1_IRQ_1;
type IrqSm = interrupt::PIO1_IRQ_1;
} }
pub type Pio0 = PioInstanceBase<0>; pub type Pio0 = PioInstanceBase<0>;