15d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner/*
25d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner * QEMU aio implementation
35d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner *
45d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner * Copyright IBM, Corp. 2008
55d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner *
65d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner * Authors:
75d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner *  Anthony Liguori   <aliguori@us.ibm.com>
85d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner *
95d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner * This work is licensed under the terms of the GNU GPL, version 2.  See
105d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner * the COPYING file in the top-level directory.
115d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner *
125d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner */
135d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
145d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner#include "qemu-common.h"
155d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner#include "block.h"
16a5d412078b8e7478d81df03710eacc7a21096ba2David 'Digit' Turner#include "qemu-queue.h"
175d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner#include "qemu_socket.h"
185d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
195d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turnertypedef struct AioHandler AioHandler;
205d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
215d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner/* The list of registered AIO handlers */
22a5d412078b8e7478d81df03710eacc7a21096ba2David 'Digit' Turnerstatic QLIST_HEAD(, AioHandler) aio_handlers;
235d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
245d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner/* This is a simple lock used to protect the aio_handlers list.  Specifically,
255d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner * it's used to ensure that no callbacks are removed while we're walking and
265d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner * dispatching callbacks.
275d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner */
285d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turnerstatic int walking_handlers;
295d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
305d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turnerstruct AioHandler
315d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner{
325d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    int fd;
335d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    IOHandler *io_read;
345d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    IOHandler *io_write;
355d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    AioFlushHandler *io_flush;
367627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner    AioProcessQueue *io_process_queue;
375d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    int deleted;
385d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    void *opaque;
39a5d412078b8e7478d81df03710eacc7a21096ba2David 'Digit' Turner    QLIST_ENTRY(AioHandler) node;
405d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner};
415d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
425d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turnerstatic AioHandler *find_aio_handler(int fd)
435d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner{
445d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    AioHandler *node;
455d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
46a5d412078b8e7478d81df03710eacc7a21096ba2David 'Digit' Turner    QLIST_FOREACH(node, &aio_handlers, node) {
475d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        if (node->fd == fd)
485d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            if (!node->deleted)
495d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                return node;
505d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    }
515d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
525d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    return NULL;
535d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner}
545d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
555d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turnerint qemu_aio_set_fd_handler(int fd,
565d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                            IOHandler *io_read,
575d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                            IOHandler *io_write,
585d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                            AioFlushHandler *io_flush,
597627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner                            AioProcessQueue *io_process_queue,
605d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                            void *opaque)
615d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner{
625d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    AioHandler *node;
635d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
645d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    node = find_aio_handler(fd);
655d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
665d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    /* Are we deleting the fd handler? */
675d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    if (!io_read && !io_write) {
685d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        if (node) {
695d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            /* If the lock is held, just mark the node as deleted */
705d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            if (walking_handlers)
715d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                node->deleted = 1;
725d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            else {
735d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                /* Otherwise, delete it for real.  We can't just mark it as
745d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                 * deleted because deleted nodes are only cleaned up after
755d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                 * releasing the walking_handlers lock.
765d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                 */
77a5d412078b8e7478d81df03710eacc7a21096ba2David 'Digit' Turner                QLIST_REMOVE(node, node);
785d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                qemu_free(node);
795d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            }
805d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        }
815d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    } else {
825d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        if (node == NULL) {
835d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            /* Alloc and insert if it's not already there */
845d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            node = qemu_mallocz(sizeof(AioHandler));
855d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            node->fd = fd;
86a5d412078b8e7478d81df03710eacc7a21096ba2David 'Digit' Turner            QLIST_INSERT_HEAD(&aio_handlers, node, node);
875d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        }
885d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        /* Update handler with latest information */
895d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        node->io_read = io_read;
905d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        node->io_write = io_write;
915d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        node->io_flush = io_flush;
927627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner        node->io_process_queue = io_process_queue;
935d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        node->opaque = opaque;
945d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    }
955d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
965d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    qemu_set_fd_handler2(fd, NULL, io_read, io_write, opaque);
975d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
985d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    return 0;
995d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner}
1005d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
1015d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turnervoid qemu_aio_flush(void)
1025d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner{
1035d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    AioHandler *node;
1045d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    int ret;
1055d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
1065d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    do {
1075d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        ret = 0;
1085d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
1095d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner	/*
1105d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner	 * If there are pending emulated aio start them now so flush
1115d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner	 * will be able to return 1.
1125d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner	 */
1135d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        qemu_aio_wait();
1145d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
115a5d412078b8e7478d81df03710eacc7a21096ba2David 'Digit' Turner        QLIST_FOREACH(node, &aio_handlers, node) {
1167627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner            if (node->io_flush) {
1177627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner                ret |= node->io_flush(node->opaque);
1187627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner            }
1195d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        }
1207627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner    } while (qemu_bh_poll() || ret > 0);
1217627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner}
1227627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner
1237627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turnerint qemu_aio_process_queue(void)
1247627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner{
1257627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner    AioHandler *node;
1267627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner    int ret = 0;
1277627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner
1287627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner    walking_handlers = 1;
1297627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner
1307627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner    QLIST_FOREACH(node, &aio_handlers, node) {
1317627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner        if (node->io_process_queue) {
1327627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner            if (node->io_process_queue(node->opaque)) {
1337627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner                ret = 1;
1347627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner            }
1357627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner        }
1367627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner    }
1377627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner
1387627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner    walking_handlers = 0;
1397627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner
1407627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner    return ret;
1415d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner}
1425d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
1435d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turnervoid qemu_aio_wait(void)
1445d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner{
1455d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    int ret;
1465d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
1475d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    if (qemu_bh_poll())
1485d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        return;
1495d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
1507627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner    /*
1517627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner     * If there are callbacks left that have been queued, we need to call then.
1527627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner     * Return afterwards to avoid waiting needlessly in select().
1537627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner     */
1547627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner    if (qemu_aio_process_queue())
1557627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner        return;
1567627ed7924456c88d3e8631c2cddcc7d54107ffaDavid Turner
1575d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    do {
1585d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        AioHandler *node;
1595d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        fd_set rdfds, wrfds;
1605d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        int max_fd = -1;
1615d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
1625d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        walking_handlers = 1;
1635d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
1645d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        FD_ZERO(&rdfds);
1655d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        FD_ZERO(&wrfds);
1665d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
1675d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        /* fill fd sets */
168a5d412078b8e7478d81df03710eacc7a21096ba2David 'Digit' Turner        QLIST_FOREACH(node, &aio_handlers, node) {
1695d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            /* If there aren't pending AIO operations, don't invoke callbacks.
1705d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner             * Otherwise, if there are no AIO requests, qemu_aio_wait() would
1715d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner             * wait indefinitely.
1725d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner             */
1735d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            if (node->io_flush && node->io_flush(node->opaque) == 0)
1745d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                continue;
1755d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
1765d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            if (!node->deleted && node->io_read) {
1775d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                FD_SET(node->fd, &rdfds);
1785d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                max_fd = MAX(max_fd, node->fd + 1);
1795d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            }
1805d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            if (!node->deleted && node->io_write) {
1815d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                FD_SET(node->fd, &wrfds);
1825d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                max_fd = MAX(max_fd, node->fd + 1);
1835d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            }
1845d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        }
1855d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
1865d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        walking_handlers = 0;
1875d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
1885d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        /* No AIO operations?  Get us out of here */
1895d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        if (max_fd == -1)
1905d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            break;
1915d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
1925d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        /* wait until next event */
1935d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        ret = select(max_fd, &rdfds, &wrfds, NULL, NULL);
1945d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        if (ret == -1 && errno == EINTR)
1955d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            continue;
1965d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
1975d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        /* if we have any readable fds, dispatch event */
1985d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        if (ret > 0) {
1995d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            walking_handlers = 1;
2005d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
2015d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            /* we have to walk very carefully in case
2025d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner             * qemu_aio_set_fd_handler is called while we're walking */
203a5d412078b8e7478d81df03710eacc7a21096ba2David 'Digit' Turner            node = QLIST_FIRST(&aio_handlers);
2045d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            while (node) {
2055d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                AioHandler *tmp;
2065d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
2075d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                if (!node->deleted &&
2085d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                    FD_ISSET(node->fd, &rdfds) &&
2095d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                    node->io_read) {
2105d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                    node->io_read(node->opaque);
2115d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                }
2125d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                if (!node->deleted &&
2135d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                    FD_ISSET(node->fd, &wrfds) &&
2145d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                    node->io_write) {
2155d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                    node->io_write(node->opaque);
2165d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                }
2175d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
2185d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                tmp = node;
219a5d412078b8e7478d81df03710eacc7a21096ba2David 'Digit' Turner                node = QLIST_NEXT(node, node);
2205d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
2215d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                if (tmp->deleted) {
222a5d412078b8e7478d81df03710eacc7a21096ba2David 'Digit' Turner                    QLIST_REMOVE(tmp, node);
2235d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                    qemu_free(tmp);
2245d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner                }
2255d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            }
2265d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner
2275d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner            walking_handlers = 0;
2285d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner        }
2295d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner    } while (ret == 0);
2305d8f37ad78fc66901af50c762029a501561f3b23David 'Digit' Turner}
231