What's Real-Time?

• Correct output
• Within deadline

Real-Time: SCHED_FIFO/SCHED_RR

                        
#define _GNU_SOURCE
#include <printf.h>
#include <pthread.h>
#include <sched.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
 
void pin_to_cpu(int cpu)
{
    cpu_set_t cpuset;
    CPU_ZERO(&cpuset);
    CPU_SET(cpu, &cpuset);
    pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
}
 
void* low_prio_thread(void* arg)
{
    pin_to_cpu(0);
    printf("low_prio: started\n");
 
    for(volatile long i = 0; i < 1e10; ++i)
        ;
 
    printf("low_prio: finished\n");
    return NULL;
}
 
void* high_prio_thread(void* arg)
{
    pin_to_cpu(0);
    // ensure low prio thread has started
    sleep(1);
    printf("hig_prio: started\n");
 
    for(volatile long i = 0; i < 1e10; ++i)
        ;
 
    printf("hig_prio: finished\n");
    return NULL;
}
 
int main()
{
    // initialize threads
    pthread_t          low, high;
    pthread_attr_t     attr;
    struct sched_param param;
    pthread_attr_init(&attr);
    pthread_attr_setschedpolicy(&attr, SCHED_FIFO);
    // pthread_attr_setschedpolicy(&attr, SCHED_OTHER);
    pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED);
 
    param.sched_priority = 10;
    pthread_attr_setschedparam(&attr, &param);
    pthread_create(&low, &attr, low_prio_thread, NULL);
 
    param.sched_priority = 30;
    pthread_attr_setschedparam(&attr, &param);
    pthread_create(&high, &attr, high_prio_thread, NULL);
 
    // cleanup
    pthread_join(low, NULL);
    pthread_join(high, NULL);
    pthread_attr_destroy(&attr);
    return 0;
}
                    

Requires root or high enough rtprio in limits.conf [3]

Priority Inversion

                        
#define _GNU_SOURCE
#include <printf.h>
#include <pthread.h>
#include <sched.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
 
pthread_mutex_t mutex;
 
void pin_to_cpu(int cpu)
{
    cpu_set_t cpuset;
    CPU_ZERO(&cpuset);
    CPU_SET(cpu, &cpuset);
    pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
}
 
void* low_prio_thread(void* arg)
{
    pin_to_cpu(0);
    printf("low_prio: started, trying to acq mutex\n");
    pthread_mutex_lock(&mutex);
    printf("low_prio: acquired mutex\n");
 
    // simulate long operation
    // both sleep and busy wait work, though differently
    // sleep(4);
    for(volatile long i = 0; i < 1e10; ++i)
        ;
 
    printf("low_prio: releasing mutex\n");
    pthread_mutex_unlock(&mutex);
    printf("low_prio: finished\n");
    return NULL;
}
 
void* high_prio_thread(void* arg)
{
    pin_to_cpu(0);
    // ensure low prio thread has lock
    sleep(1);
    printf("hig_prio: started, trying to acq mutex\n");
    pthread_mutex_lock(&mutex);
    printf("hig_prio: acquired mutex\n");
 
    printf("hig_prio: releasing mutex\n");
    pthread_mutex_unlock(&mutex);
    printf("hig_prio: finished\n");
    return NULL;
}
 
void* medium_prio_thread(void* arg)
{
    pin_to_cpu(0);
    // ensure high prio thread waits for lock
    sleep(2);
    printf("med_prio: started\n");
 
    // simulate CPU-intensive task
    // don't sleep here because we want to block the CPU
    for(volatile long i = 0; i < 1e10; ++i)
        ;
 
    printf("med_prio: finished\n");
    return NULL;
}
 
int main()
{
    // initialize mutex
    pthread_mutexattr_t mutex_attr;
    pthread_mutexattr_init(&mutex_attr);
    pthread_mutex_init(&mutex, &mutex_attr);
 
    // initialize threads
    pthread_t          low, medium, high;
    pthread_attr_t     attr;
    struct sched_param param;
    pthread_attr_init(&attr);
    pthread_attr_setschedpolicy(&attr, SCHED_FIFO);
    pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED);
 
    param.sched_priority = 10;
    pthread_attr_setschedparam(&attr, &param);
    pthread_create(&low, &attr, low_prio_thread, NULL);
 
    param.sched_priority = 30;
    pthread_attr_setschedparam(&attr, &param);
    pthread_create(&high, &attr, high_prio_thread, NULL);
 
    param.sched_priority = 20;
    pthread_attr_setschedparam(&attr, &param);
    pthread_create(&medium, &attr, medium_prio_thread, NULL);
 
    // cleanup
    pthread_join(low, NULL);
    pthread_join(medium, NULL);
    pthread_join(high, NULL);
    pthread_attr_destroy(&attr);
    pthread_mutex_destroy(&mutex);
    pthread_mutexattr_destroy(&mutex_attr);
    return 0;
}
                    

Priority Inheritance

                        
#define _GNU_SOURCE
#include <printf.h>
#include <pthread.h>
#include <sched.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
 
pthread_mutex_t mutex;
 
void pin_to_cpu(int cpu)
{
    cpu_set_t cpuset;
    CPU_ZERO(&cpuset);
    CPU_SET(cpu, &cpuset);
    pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
}
 
void* low_prio_thread(void* arg)
{
    pin_to_cpu(0);
    printf("low_prio: started, trying to acq mutex\n");
    pthread_mutex_lock(&mutex);
    printf("low_prio: acquired mutex\n");
 
    // simulate long operation
    // both sleep and busy wait work, though differently
    // sleep(4);
    for(volatile long i = 0; i < 1e10; ++i)
        ;
 
    printf("low_prio: releasing mutex\n");
    pthread_mutex_unlock(&mutex);
    printf("low_prio: finished\n");
    return NULL;
}
 
void* high_prio_thread(void* arg)
{
    pin_to_cpu(0);
    // ensure low prio thread has lock
    sleep(1);
    printf("hig_prio: started, trying to acq mutex\n");
    pthread_mutex_lock(&mutex);
    printf("hig_prio: acquired mutex\n");
 
    printf("hig_prio: releasing mutex\n");
    pthread_mutex_unlock(&mutex);
    printf("hig_prio: finished\n");
    return NULL;
}
 
void* medium_prio_thread(void* arg)
{
    pin_to_cpu(0);
    // ensure high prio thread waits for lock
    sleep(2);
    printf("med_prio: started\n");
 
    // simulate CPU-intensive task
    // don't sleep here because we want to block the CPU
    for(volatile long i = 0; i < 1e10; ++i)
        ;
 
    printf("med_prio: finished\n");
    return NULL;
}
 
int main()
{
    // initialize mutex
    pthread_mutexattr_t mutex_attr;
    pthread_mutexattr_init(&mutex_attr);
    // pthread_mutexattr_setprotocol(&mutex_attr, PTHREAD_PRIO_NONE);
    pthread_mutexattr_setprotocol(&mutex_attr, PTHREAD_PRIO_INHERIT);
    pthread_mutex_init(&mutex, &mutex_attr);
 
    // initialize threads
    pthread_t          low, medium, high;
    pthread_attr_t     attr;
    struct sched_param param;
    pthread_attr_init(&attr);
    pthread_attr_setschedpolicy(&attr, SCHED_FIFO);
    pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED);
 
    param.sched_priority = 10;
    pthread_attr_setschedparam(&attr, &param);
    pthread_create(&low, &attr, low_prio_thread, NULL);
 
    param.sched_priority = 30;
    pthread_attr_setschedparam(&attr, &param);
    pthread_create(&high, &attr, high_prio_thread, NULL);
 
    param.sched_priority = 20;
    pthread_attr_setschedparam(&attr, &param);
    pthread_create(&medium, &attr, medium_prio_thread, NULL);
 
    // cleanup
    pthread_join(low, NULL);
    pthread_join(medium, NULL);
    pthread_join(high, NULL);
    pthread_attr_destroy(&attr);
    pthread_mutex_destroy(&mutex);
    pthread_mutexattr_destroy(&mutex_attr);
    return 0;
}
                    

raw_spinlock_t

                            
static inline void __raw_spin_lock(raw_spinlock_t *lock)
{
	preempt_disable();
	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
}
 
#endif /* !CONFIG_GENERIC_LOCKBREAK || CONFIG_DEBUG_LOCK_ALLOC */
 
static inline void __raw_spin_unlock(raw_spinlock_t *lock)
{
	spin_release(&lock->dep_map, _RET_IP_);
	do_raw_spin_unlock(lock);
	preempt_enable();
}
                        

Interrupts Request Handler (IRQ)

Threaded IRQ

• No preemption in NMI, hardirq, softirq context [8]
• Threaded IRQ have RT prio 50 by default [7]

Hardware Considerations

• System management interrupts (SMI) [1]
• Simultaneous multithreading (SMT) [1]
• Dynamic frequency scaling [1]
• Much more...
  e.g., ARM Cortex-R

Sources

• [1] S. Wu, Real-time programming with Linux. [Online]. Available https://shuhaowu.com/blog/2022/03-linux-rt-appdev-part3.html
• [2] M. G. Harbour, REAL-TIME POSIX: AN OVERVIEW. [Online]. Available https://www.cs.unc.edu/~anderson/teach/comp790/papers/posix-rt.pdf
• [3] sched(7) - overview of CPU scheduling. [man page]. Available https://www.man7.org/linux/man-pages/man7/sched.7.html
• [4] pthread_mutexattr_setprotocol(3). [man page]. Available https://linux.die.net/man/3/pthread_mutexattr_setprotocol