Sys V IPC

Identifiers and Keys

• each IPC structure is referred to by a nonnegative integer identifier
• whenever an IPC structure is created, a key must be specified
• this key is converted into an identifier by the kernel

• one process creates an IPC structure by specifying a key of IPC_PRIVATE and then provides the returned identifier to the other process (e.g., by storing it in a file or forking a child, which inherits the value)
• two processes can agree on a key by defining the key value in a common header -- there is a corresponding danger that the key may already be associated with an IPC structure
• two processes can agree on a pathname and project ID (a character value between 0 and 255) and call the function ftok() to convert these values into a key

IPC creation rules

• key is IPC_PRIVATE
• key is not currently associated with an IPC structure of the particular type and the IPC_CREAT bit of flag is specified

IPC permission structure

#include <ipc.h>

struct ipc_perm {
        ushort  uid;    /* owner's user id */
        ushort  gid;    /* owner's group id */
        ushort  cuid;   /* creator's user id */
        ushort  cgid;   /* creator's group id */
        ushort  mode;   /* access modes */
        ushort  seq;    /* slot usage sequence number */
        key_t   key;    /* key */
        };

Message Queues

• long integer type
• length of the data portion of the message (can be zero)
• data (if the length is greater than zero)

#include <sys/types.h>
#include <sys/ipc.h>            /* defines the ipc_perm structure */

struct msqid_ds {
        struct ipc_perm msg_perm;       /* operation permission struct */
        struct msg      *msg_first;     /* ptr to first message on q */
        struct msg      *msg_last;      /* ptr to last message on q */
        ushort          msg_cbytes;     /* current # bytes on q */
        ushort          msg_qnum;       /* # of messages on q */
        ushort          msg_qbytes;     /* max # of bytes on q */
        ushort          msg_lspid;      /* pid of last msgsnd */
        ushort          msg_lrpid;      /* pid of last msgrcv */
        time_t          msg_stime;      /* time of last msgsnd */
        time_t          msg_rtime;      /* time of last msgrcv */
        time_t          msg_ctime;      /* time of last msgctl 
                                           (that changed the above) */
        };

Kernel representation

                struct msgid_ds
msgid ------>
                msg_perm struct   /-->  link -------->  link ---------> NULL
                msg_first -------/      type = 100      type = 200    / type = 300
                msg_last -----\         length = 1      length = 2   /  length = 3
                   ...         \        data            data        /   data
                msg_ctime       \                       ...        /    ...
                                 \                                /     ...
                                  \______________________________/

Message Queue Creation

#include <sys/types.h>
#include <sys/ipc.h>
#include <sys/msg.h>

int msgget (key_t key, int msgflag);

Message Send/Receive

#include <sys/types.h>
#include <sys/ipc.h>
#include <sys/msg.h>

#define NBYTES 512        /* size of largest message we need to send */

struct msgbuf {
    long mtype;           /* message type, must be > 0 */
    char mtext[NBYTES];   /* message data */
    };

int msgsnd (int msqid, struct msgbuf *ptr, int length, int flag);
int msgrcv (int msqid, struct msgbuf *ptr, int length, long msgtype, int flag);

• If msgtype is zero, the first message on the queue is returned.  Since each message queue is maintained as a first-in, first-out list, a msgtype of zero specifies that the oldest message on the queue is to be returned.
• If msgtype is greater than zero, the first message with a type equal to msgtype is returned.
• If msgtype is less than zero, the first message with the lowest type that is less than or equal to the absolute value of msgtype is returned.

Message Control

#include <sys/types.h>
#include <sys/ipc.h>
#include <sys/msg.h>

int msgctl (int msqid, int cmd, struct msqid_ds *buf);

Semaphores

• A SYSV semaphore consists of a set of one or more semaphore values; the number must be specified when we create it
• The creation of a semaphore (semget) is independent of its initialization (semctl); hence, we cannot atomically create and initialize a new semaphore set.  This is a fatal flaw.
• Since all forms of SYSV IPC remain in existence even when no process is using them, care must be taken to release allocated semaphores when the program is finished.

#include <sys/types.h>
#include <sys/ipc.h>            /* defines the ipc_perm structure */

struct sem {
        ushort          semval;         /* semaphore value, nonnegative */
        short           sempid;         /* pid of last operation */
        ushort          semncnt;        /* # awaiting semval > cval */
        ushort          semzcnt;        /* # awaiting semval = 0 */
        };

struct semid_ds {
        struct ipc_perm sem_perm;       /* operation permission struct */
        struct sem      *sem_base;      /* ptr to first semaphore in set */
        ushort          sem_nsems;      /* # of semaphores in set */
        time_t          sem_otime;      /* time of last semop */
        time_t          sem_ctime;      /* time of last change */
        };

Kernel representation

                struct semid_ds
semid ------>
                sem_perm struct   /-->  semval[0] 
                sem_base  -------/      sempid[0]
                sem_otime               semncnt[0]
                sem_ctime               semzcnt[0]

                                        semval[1]
                                        sempid[1]
                                        semncnt[1]
                                        semzcnt[1]

Semaphore Creation

#include <sys/types.h>
#include <sys/ipc.h>
#include <sys/sem.h>

int semget (key_t key, int nsems, int semflag);

Semaphore Wait/Signal Operations

#include <sys/types.h>
#include <sys/ipc.h>
#include <sys/sem.h>

struct sembuf {
    ushort   sem_num;   /* semaphore # */
    short    sem_op;    /* semaphore operation */
    short    sem_flg;   /* operation flags */
    };

[atomic] int semop (int semid, struct sembuf *opsptr, unsigned int nops);

• If sem_op is positive, its value is added to the semaphore's current value.  This corresponds to the release of resources that a semaphore controls.
• If sem_op is zero, the caller waits until the semaphore's value becomes zero.
• If sem_op is negative, the caller waits until the semaphore's value becomes greater than or equal to the absolute value of sem_op.  Then the absolute value is subtracted from the semaphore's current value.  This corresponds to the allocation of resources.

Semaphore Control

#include <sys/types.h>
#include <sys/ipc.h>
#include <sys/sem.h>

union semun {
    int               val;      /* used for SETVAL only */
    struct semid_ds   *buf;     /* used for IPC_STAT and IPC_SET */
    ushort            *array;   /* used for GETALL & SETALL */
    } arg;

int semctl (int semid, int semnum, int cmd, union semun arg);

Semaphore Adjustment on exit

Shared Memory

struct shmid_ds {
        struct ipc_perm shm_perm;       /* operation permission struct */
        struct anon_map *shm_amp;       /* ptr in kernel */
        int             shm_segsz;      /* size of segment in bytes */
        ushort          shm_lkcnt;      /* # of times segment is being locked */
        pid_t           shm_lpid;       /* pid of last shmop() */
        pid_t           shm_cpid;       /* pid of creater */
        ulong           shm_nattch;     /* number of current attaches */
        ulong           shm_cnattach;   /* used only for shminfo */
        time_t          shm_atime;      /* last attach time */
        time_t          shm_dtime;      /* last detach time */
        time_t          shm_ctime;      /* last change time */
        };

Obtaining A Shared Memory Segment

#include <sys/types.h>
#include <sys/ipc.h>
#include <sys/shm.h>

int shmget (key_t key, int size, int flag);

Shared Memory Control

#include <sys/types.h>
#include <sys/ipc.h>
#include <sys/shm.h>

int shmctl (int shmid, int cmd, struct shmid_ds *buf);

Shared Memory Access

#include <sys/types.h>
#include <sys/ipc.h>
#include <sys/shm.h>

int shmat (int shmid, void *addr, int flag);

• if addr is 0, the segment is attached at the first available address selected by the kernel (recommended)
• if addr is nonzero and SHM_RND is not specified, the segment is attached at the address given by addr
• if addr is nonzero and SHM_RND is specified, the segment is attached at the address given by (addr - (addr % SHMLBA).  The SHM_RND command stands for "round."

int shmdt (void *addr);

The addr argument is the value returned previously by shmat().

Example Code illustrating SYSV IPC

#include <stdio.h>
#include <sys/types.h>
#include <sys/ipc.h>
#include <sys/shm.h>
#include <sys/sem.h>
#include <sys/wait.h>
#include <sys/msg.h>

/* a record in shared memory */
typedef struct {
    int a, b;
    } SharedRec;

/* binary semaphore wait */
void semwait(int id) {
    struct sembuf buf;

    buf.sem_num = 0;
    buf.sem_op = -1;
    buf.sem_flg = 0;

    if (-1 == semop (id, &buf, 1)) {
        perror ("semop");
        exit(1);
    }
}

/* binary semaphore signal */
void semsignal (int id) {
    struct sembuf buf;

    buf.sem_num = 0;
    buf.sem_op = 1;
    buf.sem_flg = 0;

    if (-1 == semop (id, &buf, 1)) {
        perror ("semop");
        exit(1);
    }
}

/* a message */
#define MSG_SIZE 20
typedef struct {
    long type;
    char msgtxt[MSG_SIZE];
    } Message;

int main (int argc, char *argv[]) {
    int shid = 0;
    int semid;
    int qid;
    int i, j;
    SharedRec *sr;
    union semun un;
    Message msg, rcv;

    /* initialize message */
    msg.type = 1;
    strcpy (msg.msgtxt, "hello world");

    /* get a private message queue */
    if (-1 == (qid = msgget (IPC_PRIVATE, MSG_R | MSG_W))) {
        perror ("msgget");
        exit(1);
    }

    /* send a message to myself */
    if (-1 == msgsnd (qid, &msg, MSG_SIZE, 0)) {
        perror ("msgsnd");
        exit(1);
    }

    /* receive that message and print it */
    if (-1 == msgrcv (qid, &rcv, MSG_SIZE, 1, 0)) {
        perror ("msgrcv");
        exit(1);
    }
    printf ("rcv: %d - %s\n\n", rcv.type, rcv.msgtxt);

    /* 
     * NOTE: If multiple processes access this code,
     * we have a possible race condition here!
     */

    /* get a semaphore to protect shared memory area */
    if (-1 == (semid = semget (IPC_PRIVATE, 1, SEM_R | SEM_A))) {
        perror ("semget");
        exit(1);
    }

    /* initialize the semaphore */
    un.val = 1;
    if (-1 == semctl (semid, 0, SETVAL, un)) {
        perror ("semctl");
        exit(1);
    }

    /* get shared memory area for 10 SharedRec structures */
    if (-1 == (shid = shmget (IPC_PRIVATE, 10 * sizeof(SharedRec), SHM_R | SHM_W))) {
        perror ("shmget");
        exit(1);
    }

    /* attach shared memory to the record pointer */
    if (-1 == (int)(sr = (SharedRec *)shmat (shid, 0, 0))) {
        perror ("shmat");
        exit(1);
    }

    /* fork a child; write and read something in the shared area in both */
    switch (fork()) {
    case -1:
        perror ("fork");
        exit(1);

    case 0: 
        fprintf (stderr, "I am a child\n");

        for (j = 0; j < 10; j++) {
            semwait (semid);
            for (i = 0; i < 10; i++) {
                sr[i].a = sr[i].b = 2;
            }
            semsignal (semid);

            sleep(1);    /* don't want to exit too quickly */

            semwait (semid);
            for (i = 0; i < 10; i++) {
                fprintf (stderr, "\ta %d b %d\n", sr[i].a, sr[i].b);
            }
            semsignal (semid);
        }
        exit(0);

    default:
        /* same routine for the parent */
        for (j = 0; j < 10; j++) {
            semwait (semid);
            for (i = 0; i < 10; i++) {
                sr[i].a = sr[i].b = 1;
            }
            semsignal (semid);

            sleep(1);    /* don't want to exit too quickly */

            semwait (semid);
            for (i = 0; i < 10; i++) {
                fprintf (stderr, "a %d b %d\n", sr[i].a, sr[i].b);
            }
            semsignal (semid);
        }
    }

    /* clean up after child */
    wait (NULL);

    /* detach and clear shared memory */
    if (-1 == shmdt ((void *)sr)) {
        perror ("shmdt");
        exit(1);
    }

    if (-1 == shmctl (shid, IPC_RMID, NULL)) {
        perror ("shmctl");
        exit(1);
    }

    /* clear semaphore */
    if (-1 == semctl (semid, 1, IPC_RMID, un)) {
        perror ("semctl");
        exit(1);
    }

    /* clear msgq */
    if (-1 == msgctl (qid, IPC_RMID, NULL)) {
        perror ("msgctl");
        exit(1);
    }

    return 0;
}

Race-free Semaphore Initialization

• no race is possible if an IPC_PRIVATE semaphore is created/initialized by a process before fork()
• if unrelated processes access a semaphore via a common key, and include code to create/initialize the semaphore (if it does not yet exist), then a race is possible between semget() and semctl(..., SETVAL, ...)
• useful trick (Stevens) to prevent such a race is to create a semaphore set consisting of 3 semaphores for each actual semaphore needed by the program:

    semid = semget(key, 3, IPC_CREAT | 0666);
 
• key[0] is the semaphore proper, i.e., the one needed by the program to ensure mutual exclusion
• key[1] is used as a flag to indicate whether or not key[0] has been initialized
• key[2] is used to enforce mutual exclusion when setting and testing key[1]

#include <sys/types.h>
#include <sys/ipc.h>
#include <sys/sem.h>
#include <errno.h>

int sem_create(key_t key, int initval, int flags) {
    int id, semval;
    union semun sun;
    struct sembuf op_lock[2] = {
        {2, 0, 0},    /* wait for [2] (lock) to equal 0          */
        {2, 1, 0}     /* then increment [2] to 1 - this locks it */
        };
    struct sembuf op_unlock = {
        2, -1, 0      /* decrement [2] (lock) back to 0          */
        };

    if (key == IPC_PRIVATE || key == (key_t)-1) {
        errno = EINVAL;    /* invalid semaphore key */
        return -1;
    }

    again:    /* label for goto */
    if (-1 == (id = semget(key, 3, flags | IPC_CREAT))) {
        return -1;
    }

    /*
     * When the 3-member semaphore is created, we know that the value of all 
     * 3 members is 0.  Get a lock on the semaphore by waiting for [2] to equal 0, 
     * then increment it.
     *
     * There is a race condition here.  Between semget() above and semop() below,
     * another process can remove the semaphore if that process is the last one
     * to use it.  Therewfore, we have to handle the error condition of an invalid
     * semaphore ID, which we do simply by recreating it (at the again label).
     */
    if (-1 == semop(id, op_lock, 2)) {
        if (EINVAL == errno) {
            goto again;
        }
        return -1;
    }

    if (-1 == (semval = semctl(id, 1, GETVAL, 0))) {
        return -1;
    }
    if (!semval) {
        /* the semaphore has not been intialized yet */
        sun.val = initval;

        if (-1 == semctl(id, 0, SETVAL, sun)) {
            return -1;
        }
        sun.val - 1;
        if (-1 == semctl(id, 1, SETVAL, sun)) {
            return -1;
        }
    }

    /* release the lock */
    if (-1 == semop(id, &op_unlock, 1)) {
        return -1;
    }

    return id;
}