diff --git a/src/include/sof/audio/pipeline.h b/src/include/sof/audio/pipeline.h index d80e4380aa82..d604e347fb7f 100644 --- a/src/include/sof/audio/pipeline.h +++ b/src/include/sof/audio/pipeline.h @@ -129,6 +129,10 @@ struct pipeline_task { struct task task; /**< parent structure */ bool registrable; /**< should task be registered on irq */ struct comp_dev *sched_comp; /**< pipeline scheduling component */ + /* private data used by Zephyr DMA domain for internal + * housekeeping - don't touch. + */ + void *priv; }; #define pipeline_task_get(t) container_of(t, struct pipeline_task, task) diff --git a/src/include/sof/schedule/ll_schedule_domain.h b/src/include/sof/schedule/ll_schedule_domain.h index f555c256398b..126939f168fc 100644 --- a/src/include/sof/schedule/ll_schedule_domain.h +++ b/src/include/sof/schedule/ll_schedule_domain.h @@ -195,13 +195,10 @@ static inline void domain_disable(struct ll_schedule_domain *domain, int core) static inline bool domain_is_pending(struct ll_schedule_domain *domain, struct task *task, struct comp_dev **comp) { - bool ret; - - assert(domain->ops->domain_is_pending); - - ret = domain->ops->domain_is_pending(domain, task, comp); - - return ret; + if (domain->ops->domain_is_pending) + return domain->ops->domain_is_pending(domain, task, comp); + else + return true; } diff --git a/src/schedule/zephyr_dma_domain.c b/src/schedule/zephyr_dma_domain.c index 41b01e3d6200..eb4eb7345778 100644 --- a/src/schedule/zephyr_dma_domain.c +++ b/src/schedule/zephyr_dma_domain.c @@ -52,6 +52,31 @@ struct zephyr_dma_domain_data { struct dma_chan_data *channel; /* channel data */ bool enabled_irq; /* true if DMA IRQ was already enabled */ struct zephyr_dma_domain_thread *dt; + /* set to true by the DMA IRQ handler when an IRQ for + * associated channel was triggered. + */ + bool triggered; + /* used to keep track of how many pipeline tasks point + * to the same data structure. + */ + int usage_count; + /* used to keep track of how many times data->triggered + * can be left set to true before resetting it to false. + * + * this is needed for mixer topologies because the + * non-registrable pipeline tasks need to be executed + * when an IRQ comes for the registrable pipeline task. + */ + int pending_count; + /* used when linking a data structure to a non-registrable + * pipeline task. + * + * in the case of mixer, the non-registrable pipeline tasks + * have the same scheduling component as the registrable + * pipeline task so this can be used as a criteria to attach + * the data structure to the non-registrable pipeline task. + */ + struct comp_dev *sched_comp; }; struct zephyr_dma_domain_thread { @@ -80,11 +105,14 @@ static int zephyr_dma_domain_unregister(struct ll_schedule_domain *domain, uint32_t num_tasks); static void zephyr_dma_domain_task_cancel(struct ll_schedule_domain *domain, struct task *task); +static bool zephyr_dma_domain_is_pending(struct ll_schedule_domain *domain, + struct task *task, struct comp_dev **comp); static const struct ll_schedule_domain_ops zephyr_dma_domain_ops = { .domain_register = zephyr_dma_domain_register, .domain_unregister = zephyr_dma_domain_unregister, - .domain_task_cancel = zephyr_dma_domain_task_cancel + .domain_task_cancel = zephyr_dma_domain_task_cancel, + .domain_is_pending = zephyr_dma_domain_is_pending, }; struct ll_schedule_domain *zephyr_dma_domain_init(struct dma *dma_array, @@ -143,13 +171,21 @@ static void dma_irq_handler(void *data) irq = zephyr_dma_domain_data->irq; dt = zephyr_dma_domain_data->dt; - /* clear IRQ */ - dma_interrupt_legacy(channel, DMA_IRQ_CLEAR); - interrupt_clear_mask(irq, BIT(channel_index)); + /* was the IRQ triggered by this channel? */ + if (dma_interrupt_legacy(channel, DMA_IRQ_STATUS_GET)) { + /* IRQ is for this channel. Make sure that the scheduler + * can schedule the associated pipeline task. + */ + zephyr_dma_domain_data->triggered = true; - /* give resources to thread semaphore */ - if (dt->handler) - k_sem_give(sem); + /* clear IRQ */ + dma_interrupt_legacy(channel, DMA_IRQ_CLEAR); + interrupt_clear_mask(irq, BIT(channel_index)); + + /* give resources to thread semaphore */ + if (dt->handler) + k_sem_give(sem); + } } static int enable_dma_irq(struct zephyr_dma_domain_data *data) @@ -204,6 +240,24 @@ static int register_dma_irq(struct zephyr_dma_domain *domain, if (core != crt_chan->core) continue; + if ((crt_data->sched_comp == pipe_task->sched_comp || + !crt_data->sched_comp) && !pipe_task->registrable) { + /* attach crt_data to current pipeline + * task only if there's a match between + * the scheduling components or i + * crt_data's sched_comp field wasn't + * set yet. + */ + pipe_task->priv = crt_data; + + crt_data->usage_count++; + crt_data->pending_count++; + + return 0; + } else if (!pipe_task->registrable) { + return 0; + } + /* skip if IRQ was already registered */ if (crt_data->enabled_irq) continue; @@ -215,6 +269,11 @@ static int register_dma_irq(struct zephyr_dma_domain *domain, crt_data->irq = irq; crt_data->channel = crt_chan; crt_data->dt = dt; + crt_data->sched_comp = pipe_task->sched_comp; + crt_data->usage_count++; + crt_data->pending_count++; + + pipe_task->priv = crt_data; if (dt->started) { /* if the Zephyr thread was started, we @@ -261,16 +320,6 @@ static int zephyr_dma_domain_register(struct ll_schedule_domain *domain, tr_info(&ll_tr, "zephyr_dma_domain_register()"); - /* don't even bother trying to register DMA IRQ for - * non-registrable tasks. - * - * this is needed because zephyr_dma_domain_register() will - * return -EINVAL for non-registrable tasks because of - * register_dma_irq() which is not right. - */ - if (!pipe_task->registrable) - return 0; - /* the DMA IRQ has to be registered before the Zephyr thread is * started. * @@ -285,12 +334,16 @@ static int zephyr_dma_domain_register(struct ll_schedule_domain *domain, if (ret < 0) { tr_err(&ll_tr, - "failed to register DMA IRQ for pipe task %p on core %d", - pipe_task, core); + "failed to register DMA IRQ for pipe task %p on core %d reg: %d", + pipe_task, core, pipe_task->registrable); return ret; } + /* don't let non-registrable tasks go further */ + if (!pipe_task->registrable) + return 0; + /* skip if Zephyr thread was already started on this core */ if (dt->handler) return 0; @@ -429,15 +482,20 @@ static int zephyr_dma_domain_unregister(struct ll_schedule_domain *domain, struct zephyr_dma_domain *zephyr_dma_domain; struct zephyr_dma_domain_thread *dt; struct pipeline_task *pipe_task; + struct zephyr_dma_domain_data *data; int ret, core; zephyr_dma_domain = ll_sch_get_pdata(domain); pipe_task = pipeline_task_get(task); core = cpu_get_id(); dt = zephyr_dma_domain->domain_thread + core; + data = pipe_task->priv; tr_info(&ll_tr, "zephyr_dma_domain_unregister()"); + data->pending_count--; + data->usage_count--; + /* unregister the DMA IRQ only for PPL tasks marked as "registrable" * * this is done because, in case of mixer topologies there's @@ -491,3 +549,33 @@ static void zephyr_dma_domain_task_cancel(struct ll_schedule_domain *domain, k_sem_give(&dt->sem); } } + +static bool zephyr_dma_domain_is_pending(struct ll_schedule_domain *domain, + struct task *task, struct comp_dev **comp) +{ + struct zephyr_dma_domain_data *data; + struct pipeline_task *pipe_task; + + pipe_task = pipeline_task_get(task); + data = pipe_task->priv; + + /* note: there's no need to disable IRQs here as they already + * are inside zephyr_ll_run when this function is called. + * + * VERY IMPORTANT: if this function needs to be moved outside + * of the atomic area PLEASE make sure to disable IRQs before + * calling it or disable IRQs here. + */ + if (data->triggered) { + data->pending_count--; + + if (!data->pending_count) { + data->triggered = false; + data->pending_count = data->usage_count; + } + + return true; + } + + return false; +} diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 7405dc0b22a9..cc195791f347 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -195,6 +195,10 @@ static void zephyr_ll_run(void *data) list_item_del(list); list_item_append(list, &task_head); + /* check if the task can be scheduled */ + if (!domain_is_pending(sch->ll_domain, task, NULL)) + continue; + zephyr_ll_unlock(sch, &flags); /*