text:
/* The serial driver connects UART HW to mbed and also associates the UART
 * HW with physical pins.  Any physical pin can be linked to any UART,
 * however the mbed serial port initialisation API makes no mention of
 * which UART HW is to be used (only the pins) and hence the driver needs
 * to make some decisions for itself.
 *
 * There are two and a half UARTs on the chip: UART0, UART1 and a
 * lower-power, receive-only UART that is clocked from 32 kHz and can
 * therefore be awake while the rest of the chip is sleeping peacefully.
 * This provides maximal power saving, however the LP UART can only run
 * at 9600 bits/s (which is quite sufficient for all NB-IoT needs).
 *
 * So, if the baud rate is 9600 the driver code configures the LP UART
 * for Rx and UART0 for Tx. If the baud rate is not 9600 then it configures
 * UART0 for both Rx and Tx.  Unless... the Tx pin is the pin UART1_TX (it
 * is an mbed convention to use the Tx pin), which is p6, in which case UART1
 * is configured instead.  This latter is not the normal case as this pin
 * is intended to be used as a GPIO.
 *
 * If the baud rate is changed the driver reconfigures to match.
 *
 * TODO: implement asynchronous and flow control APIs.
 */

#include "mbed_assert.h"
#include "serial_api.h"
#include "pinmap.h"

#include "cmsis.h"

/* ----------------------------------------------------------------
 * MACROS
 * ----------------------------------------------------------------*/

/* Registers banks for the standard UARTs */
#define UART0_REG (*(volatile uart_ctrl_t *) UART0_BASE)
#define UART1_REG (*(volatile uart_ctrl_t *) UART1_BASE)

/* Masks for the UART control bits in the reset and clock enable registers */
#define UART0_CTRL  (1 << 3)
#define UART1_CTRL  (1 << 4)
#define UARTLP_CTRL (1 << 6)

/* Convert number of data bits to register values */
#define MIN_NUM_UART_DATA_BITS 5
#define MAX_NUM_UART_DATA_BITS 8
#define REGISTER_DATA_BITS(x) ((x) - MIN_NUM_UART_DATA_BITS)

/* Number of stop bits */
#define NUM_UART_STOP_BITS_1 1
#define NUM_UART_STOP_BITS_2 2

/* ----------------------------------------------------------------
 * TYPES
 * ----------------------------------------------------------------*/

/* Enum to identify the interrupt to the UART handler */
typedef enum {
    IRQ_UART_ID_0_AND_LP,
    IRQ_UART_ID_1,
    NUM_IRQ_IDS
} irq_uart_id_t;

/* ----------------------------------------------------------------
 * GLOBAL VARIABLES
 * ----------------------------------------------------------------*/

/* The IRQ configuration variables, set up and named by mbed */
static uint32_t serial_irq_ids[NUM_IRQ_IDS] = {0};
static uart_irq_handler irq_handler = NULL;

/* RTX needs these */
int stdio_uart_inited = 0;
serial_t stdio_uart;

/* ----------------------------------------------------------------
 * FUNCTION PROTOTYPES
 * ----------------------------------------------------------------*/

static void init_config(serial_t *obj);
static void deinit_config(serial_t *obj);
static void set_baud(serial_t *obj, uint32_t baud_rate);
static void irq_enable(serial_t *obj);
static void irq_disable(serial_t *obj);

/* ----------------------------------------------------------------
 * NON-API FUNCTIONS
 * ----------------------------------------------------------------*/

/* Initialise the given serial config by setting the pin functions
 * and then resetting the relevant HW */
static void init_config(serial_t *obj)
{
    uint32_t x;

    switch (obj->config) {
        case SERIAL_CONFIG_UARTLP_RX_UART0_TX:
        {
            pin_function(obj->rx_pin, PIN_FUNCTION_LP_UART);
            pin_function(obj->tx_pin, PIN_FUNCTION_UART0_TXD);
            CLKEN_REG_BITSET = UARTLP_CTRL | UART0_CTRL;
            obj->reg_base = &UART0_REG;
            obj->index = IRQ_UART_ID_0_AND_LP;
            /* Reset the LPUART and UART0 HW */
            /* NOTE: RESET_REG_BITTOG doesn't have the desired
             * effect, need to use BITSET and then BITCLR */
            RESET_REG_BITSET |= 1ul << 6;
            RESET_REG_BITCLR |= 1ul << 6;
            RESET_REG_BITSET |= 1ul << 3;
            RESET_REG_BITCLR |= 1ul << 3;
        }
        break;
        case SERIAL_CONFIG_UART0_RX_UART0_TX:
        {
            pin_function(obj->rx_pin, PIN_FUNCTION_UART0_RXD);
            pin_function(obj->tx_pin, PIN_FUNCTION_UART0_TXD);
            CLKEN_REG_BITSET = UART0_CTRL;
            obj->reg_base = &UART0_REG;
            obj->index = IRQ_UART_ID_0_AND_LP;
            /* Reset the UART0 HW */
            RESET_REG_BITSET |= 1ul << 3;
            RESET_REG_BITCLR |= 1ul << 3;
        }
        break;
        case SERIAL_CONFIG_UART1_RX_UART1_TX:
        {
            pin_function(obj->rx_pin, PIN_FUNCTION_UART1_RXD);
            pin_function(obj->tx_pin, PIN_FUNCTION_UART1_TXD);
            CLKEN_REG_BITSET = UART1_CTRL;
            obj->reg_base = &UART1_REG;
            obj->index = IRQ_UART_ID_1;
            /* Reset the UART1 HW */
            RESET_REG_BITSET |= 1ul << 4;
            RESET_REG_BITCLR |= 1ul << 4;
        }
        break;
        default:
        {
            MBED_ASSERT(false);
        }
        break;
    }

    /* Tickle the UART control register to make sure it is updated */
    x = obj->reg_base->UARTLCR_H;
    obj->reg_base->UARTLCR_H = x;

    /* Set the FIFO. The meaning of the three FIFO interrupt-level
     * bits are as follows:
     *
     * 0 = 1/8 full
     * 1 = 1/4 full
     * 2 = 1/2 full
     * 3 = 3/4 full
     * 4 = 7/8 full
     *
     * Set up the Rx FIFO to be used fully (but we will also set
     * a timeout to get immediate notice) and also the Tx FIFO
     * to be fully used. */
     obj->reg_base->UARTIFLS = (obj->reg_base->UARTIFLS & ~(0x07 << 0)) | (4 << 0);
     obj->reg_base->UARTIFLS = (obj->reg_base->UARTIFLS & ~(0x07 << 3)) | (4 << 3);
     obj->reg_base->UARTLCR_H |= 1 << 4;

    /* Enable for Tx and Rx (TODO: add CTS when we add flow control) */
    obj->reg_base->UARTCR |= (1 << 8) | (1 << 9);

    /* Now enable it */
    obj->reg_base->UARTCR |= 1 << 0;

    obj->format_set = false;
    obj->baud_rate = 0;
    obj->irq_rx_setting = IRQ_NOT_SET;
    obj->irq_tx_setting = IRQ_NOT_SET;
}

/* Release a serial port */
static void deinit_config(serial_t *obj)
{
    pin_function(obj->rx_pin, PIN_FUNCTION_UNCLAIMED);
    pin_function(obj->tx_pin, PIN_FUNCTION_UNCLAIMED);

    /* Disable UART */
    obj->reg_base->UARTCR &= ~(1 << 0);

    /* Flush transmit FIFO */
    obj->reg_base->UARTLCR_H = 0;

    /* Disable everything */
    obj->reg_base->UARTCR = 0;

    switch (obj->config) {
        case SERIAL_CONFIG_UARTLP_RX_UART0_TX:
        {
            CLKEN_REG_BITCLR = UARTLP_CTRL | UART0_CTRL;
            LP_UART_CTRL &= ~(0xF << 20); /* Disable all LP interrupts */
        }
        break;
        case SERIAL_CONFIG_UART0_RX_UART0_TX:
        {
            CLKEN_REG_BITCLR = UART0_CTRL;
        }
        break;
        case SERIAL_CONFIG_UART1_RX_UART1_TX:
        {
            CLKEN_REG_BITCLR = UART1_CTRL;
        }
        break;
        default:
        {
            MBED_ASSERT(false);
        }
        break;
    }

    obj->config = MAX_NUM_SERIAL_CONFIGS;
    obj->reg_base = NULL;
}

/* Set the baud rate for either of the two (non-LP) UARTS */
static void set_baud(serial_t *obj, uint32_t baud_rate)
{
    uint32_t baud_rate_divider_i;
    uint32_t baud_rate_divider_f;
    uint32_t remainder;
    uint32_t core_clock;
    uint32_t x;

    /* Baud rate divider calculation:
     *
     * The integer part is: clock / (16 * baud)
     *
     * The fractional part is:  round (decimal_part * 64),
     * ...where decimal part is, for example, 0.085
     *
     * decimal_part is: remainder / (16 * baud),
     * ...where: remainder = core_clock % (baud * 16),
     *
     * So the fractional part becomes:
     * round (decimal_part * 64) = round (remainder * 64 / (16 * baud)) = round (remainder * 4 / baud)
     */

    /* Get the mean clock frequency */
    core_clock = (CLK_FREQ_HIGHTARGET >> 1) + (CLK_FREQ_LOWTARGET >> 1);
    /* Work out the actual clock frequency */
    core_clock = (core_clock * CLOCKS_REFERENCE_CLOCK_FREQ) / (((CLK_FREQ_NREFCLKS + 1) & 0xFFFF) * (CLK_GATE_SYS & 0xFF));
    baud_rate_divider_i = core_clock / (baud_rate << 4);
    remainder = core_clock % (baud_rate << 4);
    baud_rate_divider_f = ((remainder << 3) / baud_rate) >> 1;
    /* Round it */
    baud_rate_divider_f += ((remainder << 3) / baud_rate) & 1;

    /* Disable UART while writing to control registers */
    obj->reg_base->UARTCR &= ~(1 << 0);

    obj->reg_base->UARTIBRD = baud_rate_divider_i;
    obj->reg_base->UARTFBRD = baud_rate_divider_f;

    /* Make IBRD and FBRD update */
    x = obj->reg_base->UARTLCR_H;
    obj->reg_base->UARTLCR_H = x;

    /* Now enable the UART again */
    obj->reg_base->UARTCR |= 1 << 0;
}

/* Set the NVIC bits */
static void irq_enable(serial_t *obj)
{
    switch (obj->config) {
        case SERIAL_CONFIG_UARTLP_RX_UART0_TX:
        {
            NVIC_EnableIRQ(UART0_IRQn);
            NVIC_EnableIRQ(LPUART_IRQn);
        }
        break;
        case SERIAL_CONFIG_UART0_RX_UART0_TX:
        {
            NVIC_EnableIRQ(UART0_IRQn);
        }
        break;
        case SERIAL_CONFIG_UART1_RX_UART1_TX:
        {
            NVIC_EnableIRQ(UART1_IRQn);
        }
        break;
        default:
        {
            MBED_ASSERT(false);
        }
        break;
    }
}

/* Unset the NVIC bits */
static void irq_disable(serial_t *obj)
{
    switch (obj->config) {
        case SERIAL_CONFIG_UARTLP_RX_UART0_TX:
        {
            NVIC_DisableIRQ(UART0_IRQn);
            NVIC_DisableIRQ(LPUART_IRQn);
        }
        break;
        case SERIAL_CONFIG_UART0_RX_UART0_TX:
        {
            NVIC_DisableIRQ(UART0_IRQn);
        }
        break;
        case SERIAL_CONFIG_UART1_RX_UART1_TX:
        {
            NVIC_DisableIRQ(UART1_IRQn);
        }
        break;
        default:
        {
            MBED_ASSERT(false);
        }
        break;
    }
}

/* UART0 IRQ */
void IRQ7_UART0_Handler()
{
    uint32_t id = serial_irq_ids[IRQ_UART_ID_0_AND_LP];

    /* Check Rx and Rx Timeout bit */
    if (UART0_REG.UARTMIS & ((1 << 4) | (1 << 6))) {
        if (id != 0) {
            irq_handler(id, RxIrq);
            /* Reading the character clears the interrupt,
             * no way to protect against another arriving
             * while processing one */
        }
    }
    /* Check Tx bit */
    if (UART0_REG.UARTMIS & (1 << 5)) {
        if (id != 0) {
            irq_handler(id, TxIrq);
        }
        /* Not sure what clears the interrupt so clear it explicitly */
        NVIC_ClearPendingIRQ(UART1_IRQn);
    }
}

/* UART1 IRQ */
void IRQ8_UART1_Handler()
{
    uint32_t id = serial_irq_ids[IRQ_UART_ID_1];

    /* Check Rx and Rx Timeout bit */
    if (UART1_REG.UARTMIS & ((1 << 4) | (1 << 6))) {
        if (id != 0) {
            irq_handler(id, RxIrq);
        }
        /* Reading the character clears the interrupt,
         * no way to protect against another arriving
         * while processing one */
    }
    /* Check Tx bit */
    if (UART1_REG.UARTMIS & (1 << 5)) {
        if (id != 0) {
            irq_handler(id, TxIrq);
        }
        /* Not sure what clears the interrupt so clear it explicitly */
        NVIC_ClearPendingIRQ(UART1_IRQn);
    }
}

/* LP UART IRQ, receive only */
void IRQ16_LPUART_Handler()
{
    uint32_t id = serial_irq_ids[IRQ_UART_ID_0_AND_LP];

    if (id != 0) {
        irq_handler(id, RxIrq);

        /* Another character might have arrived while
         * we are processing the last, so
         * check status bits 8 to 10 again and pend
         * interrupt if there's something there */
        if (((LP_UART_STATUS >> 8) & 0x07) != 0) {
            NVIC_SetPendingIRQ(LPUART_IRQn);
        } else {
            LP_UART_CTRL |= 1 << 27; /* Clear the interrupt */
        }
    }
}

/* ----------------------------------------------------------------
 * MBED API CALLS: SETUP FUNCTIONS
 * ----------------------------------------------------------------*/

void serial_init(serial_t *obj, PinName tx, PinName rx)
{
    uint32_t clock = CLK_FREQ_DIG_CLKS;

    /* There are two and a half UARTs on the chip. The normal
     * configuration is to use the LP_UART for Rx and UART0 for
     * Tx.  This gives maximal power saving in that the chip can
     * wake up on receipt of data. However, this only works if the
     * data rate is 9600 because that's the only data rate that
     * the 32 kHz (i.e. RTC) clock driving the LP UART can support.
     *
     * So, if the data rate is 9600, use the LP_UART/UART0
     * combination, otherwise use UART0 for both Rx and Tx.  However,
     * we don't know the data rate at this point so assume LP_UART
     * (as this works at the default baud rate) and we can change
     * our minds later.
     *
     * There is another serial port, UART1, which is normally used
     * by the modem processor to send out debug.  We only initialise
     * that here if the Tx pin is UART1_TX. */

    /* Wait for the clock to be stable */
    while ((clock < CLK_FREQ_LOWTARGET) || (clock > CLK_FREQ_HIGHTARGET)) {
        clock = CLK_FREQ_DIG_CLKS;
    }

    if (tx == UART1_TX) {
        /* Use UART1 for Rx and Tx */
        obj->config = SERIAL_CONFIG_UART1_RX_UART1_TX;
    } else {
        /* Use LP_UART for Rx, UART0 for Tx */
        obj->config = SERIAL_CONFIG_UARTLP_RX_UART0_TX;
    }

    obj->rx_pin = rx;
    obj->tx_pin = tx;
    init_config(obj);

    /* TODO: set rx pin Pull mode ? */

    /* set default baud rate and format */
    serial_baud(obj, 9600);
    serial_format(obj, 8, ParityNone, 1);

    if (tx == UART0_TX) {
        /* The UART0 pins are the stdio pins */
        stdio_uart_inited = 1;
        stdio_uart = *obj;
    }
}

void serial_free(serial_t *obj)
{
    if (obj->tx_pin == UART0_TX) {
        stdio_uart_inited = 0;
    }

    serial_irq_ids[obj->index] = 0;

    /* Release the port HW */
    deinit_config(obj);
}

void serial_baud(serial_t *obj, int baudrate)
{
    bool switch_port_config = false;
    bool format_set = obj->format_set;
    uint8_t stop_bits = obj->format.stop_bits;
    uint8_t data_bits = obj->format.data_bits;
    SerialParity parity = (SerialParity) obj->format.parity;
    irq_setting_t irq_rx_setting = obj->irq_rx_setting;
    irq_setting_t irq_tx_setting = obj->irq_tx_setting;

    if ((obj->config == SERIAL_CONFIG_UARTLP_RX_UART0_TX) && (baudrate != 9600)) {
        /* If we were on LP UART but the baud rate is not 9600 then
         * switch to the standard UART (as the LP UART can't go any higher
         * because it's clocked from 32 kHz) */
        deinit_config(obj);
        obj->config = SERIAL_CONFIG_UART0_RX_UART0_TX;
        init_config(obj);
        switch_port_config = true;
    } else if ((obj->config == SERIAL_CONFIG_UART0_RX_UART0_TX) && (baudrate == 9600)) {
        /* If we were on UART0 but the baud rate is 9600 then switch to the
         * LP UART for receive */
        deinit_config(obj);
        obj->config = SERIAL_CONFIG_UARTLP_RX_UART0_TX;
        init_config(obj);
        switch_port_config = true;
    }

    /* Disable UART while writing to control registers */
    obj->reg_base->UARTCR &= ~(1 << 0);

    if (switch_port_config) {
        /* If the port was switched, switch the port configuration also */
        if (format_set) {
            /* This serial port has been previously set up so switch the
             * settings across to this new configuration */
            serial_format(obj, data_bits, parity, stop_bits);
        }
        if (irq_rx_setting != IRQ_NOT_SET) {
            /* Do the same for Rx interrupts, if they were set */
            serial_irq_set(obj, RxIrq, (irq_rx_setting == IRQ_ON));
        }
        if (irq_tx_setting != IRQ_NOT_SET) {
            /* Do the same for Tx interrupts, if they were set */
            serial_irq_set(obj, TxIrq, (irq_tx_setting == IRQ_ON));
        }
    }

    switch (obj->config) {
        case SERIAL_CONFIG_UARTLP_RX_UART0_TX:
        {
            /* Set LP UART to 9600 (numerator 75 (0x4B), denominator 256 (00 == 256)) */
            LP_UART_CTRL = (LP_UART_CTRL & ~0xFFFF) | 0x004B;
            set_baud(obj, baudrate);
        }
        break;
        case SERIAL_CONFIG_UART0_RX_UART0_TX:
        case SERIAL_CONFIG_UART1_RX_UART1_TX:
        {
            set_baud(obj, baudrate);
        }
        break;
        default:
        {
            MBED_ASSERT(false);
        }
        break;
    }

    /* Enable the UART again */
    obj->reg_base->UARTCR |= 1 << 0;

    obj->baud_rate = baudrate;
}

void serial_format(serial_t *obj, int data_bits, SerialParity parity, int stop_bits)
{
    bool lp_also = false;

    MBED_ASSERT(data_bits >= MIN_NUM_UART_DATA_BITS);
    MBED_ASSERT(data_bits <= MAX_NUM_UART_DATA_BITS);
    MBED_ASSERT(stop_bits >= NUM_UART_STOP_BITS_1);
    MBED_ASSERT(stop_bits <= NUM_UART_STOP_BITS_2);

    /* The LP UART is different to UARTs 0 and 1 so deal with it
     * explicitly when required */
    if (obj->config == SERIAL_CONFIG_UARTLP_RX_UART0_TX) {
        lp_also = true;
    }

    /* Disable UART while writing to control registers */
    obj->reg_base->UARTCR &= ~(1 << 0);

    /* Set data bits (bits 5 and 6 of the UART0/1 register, bits 18 and 19 of the LP UART register) */
    obj->reg_base->UARTLCR_H = (obj->reg_base->UARTLCR_H & ~(0x03 << 5)) | (REGISTER_DATA_BITS(data_bits) << 5);
    if (lp_also) {
        LP_UART_CTRL = (LP_UART_CTRL & ~(0x03 << 18)) | (REGISTER_DATA_BITS(data_bits) << 18);
    }
    obj->format.data_bits = (uint8_t) data_bits;

    /* Set stop bits (bit 7 of the UART0/1 register) (there is no such setting for the LP UART) */
    if (stop_bits == NUM_UART_STOP_BITS_1) {
        /* Clear 2-stop-bits bit */
        obj->reg_base->UARTLCR_H &= ~(1 << 7);
    } else {
        /* Set 2-stop-bits bit */
        obj->reg_base->UARTLCR_H |= 1 << 7;
    }
    obj->format.stop_bits = (uint8_t) stop_bits;

    /* Set parity */
    switch (parity) {
        case ParityNone:
        {
            /* Disable parity */
            obj->reg_base->UARTLCR_H &= ~0x02;
            if (lp_also)
            {
                LP_UART_CTRL &= ~(1 << 24);
            }
        }
        break;
        case ParityOdd:
        {
            /* Set even bit and enable parity */
            obj->reg_base->UARTLCR_H = (obj->reg_base->UARTLCR_H | (1 << 3)) | (1 << 2);
            if (lp_also)
            {
                LP_UART_CTRL |= (1 << 24) | (1 << 25);
            }
        }
        break;
        case ParityEven:
        {
            /* Clear even bit and enable parity */
            obj->reg_base->UARTLCR_H = (obj->reg_base->UARTLCR_H & ~(1 << 3)) | (1 << 2);
            if (lp_also)
            {
                LP_UART_CTRL &= ~(1 << 25);
                LP_UART_CTRL |= 1 << 24;
            }
        }
        break;
        default:
        {
            MBED_ASSERT(false);
        }
        break;
    }

    /* Enable the UART again */
    obj->reg_base->UARTCR |= 1 << 0;

    obj->format.parity = (uint8_t) parity;
    obj->format_set = true;
}

/* ----------------------------------------------------------------
 * MBED API CALLS: INTERRUPT FUNCTIONS
 * ----------------------------------------------------------------*/

void serial_irq_handler(serial_t *obj, uart_irq_handler handler, uint32_t id)
{
    irq_handler = handler;
    serial_irq_ids[obj->index] = id;
}

void serial_irq_set(serial_t *obj, SerialIrq irq, uint32_t enable)
{
    bool lp_also = false;

    if (obj->config == SERIAL_CONFIG_UARTLP_RX_UART0_TX) {
        lp_also = true;
    }

    /* Disable UART while writing to control registers */
    obj->reg_base->UARTCR &= ~(1 << 0);

    if (enable) {
        switch (irq) {
            case RxIrq:
            {
                /* Bit 4 for Rx and bit 6 for Rx Timeout */
                obj->reg_base->UARTIMSC |= (1 << 4) | (1 << 6);
                if (lp_also)
                {
                    /* "Word Received" IRQ */
                    LP_UART_CTRL |= 1 << 23;
                }
                obj->irq_rx_setting = IRQ_ON;
                irq_enable(obj);
            }
            break;
            case TxIrq:
            {
                /* Bit 5 */
                obj->reg_base->UARTIMSC |= 1 << 5;
                obj->irq_tx_setting = IRQ_ON;
                irq_enable(obj);
            }
            break;
            default:
            {
                MBED_ASSERT(false);
            }
            break;
        }
    } else {
        switch (irq) {
            case RxIrq:
            {
                /* Bit 4  for Rx and bit 6 for Rx Timeout */
                obj->reg_base->UARTIMSC &= ~((1 << 4) | (1 << 6));
                if (lp_also)
                {
                    /* "Word Received" IRQ */
                    LP_UART_CTRL &= ~(1 << 23);
                }
                obj->irq_rx_setting = IRQ_OFF;
            }
            break;
            case TxIrq:
            {
                /* Bit 5 */
                obj->reg_base->UARTIMSC &= ~(1 << 5);
                obj->irq_tx_setting = IRQ_OFF;
            }
            break;
            default:
            {
                MBED_ASSERT(false);
            }
            break;
        }

        if ((obj->irq_rx_setting == IRQ_OFF) && (obj->irq_tx_setting == IRQ_OFF)) {
            irq_disable(obj);
        }
    }

    /* Enable the UART again */
    obj->reg_base->UARTCR |= 1 << 0;
}

/* ----------------------------------------------------------------
 * MBED API CALLS: TRANSMIT AND RECEIVE FUNCTIONS
 * ----------------------------------------------------------------*/

int serial_getc(serial_t *obj)
{
    uint8_t data = 0;

    /* Block until there is data to read */
    while (!serial_readable(obj)) {}

    /* Read the data */
    switch (obj->config) {
        case SERIAL_CONFIG_UARTLP_RX_UART0_TX:
        {
            data = (uint8_t) LP_UART_DATA;
        }
        break;
        case SERIAL_CONFIG_UART0_RX_UART0_TX:
        case SERIAL_CONFIG_UART1_RX_UART1_TX:
        {
            data = (uint8_t) obj->reg_base->UARTDR;
        }
        break;
        default:
        {
            MBED_ASSERT(false);
        }
        break;
    }

    return (int) data;
}

void serial_putc(serial_t *obj, int c)
{
    /* Block until there is room to write */
    while (!serial_writable(obj)) {}

    /* Write the data */
    obj->reg_base->UARTDR = (uint8_t) c;
}

int serial_readable(serial_t *obj)
{
    bool readable = false;

    switch (obj->config) {
        case SERIAL_CONFIG_UARTLP_RX_UART0_TX:
        {
            /* Check the status register, bits 8 to 10 indicate
             * the number of Rx bytes (make sure it's the status
             * register not the data register as a read from that
             * register would clear the Rx interrupt) */
            readable = (((LP_UART_STATUS >> 8) & 0x07) != 0);
        }
        break;
        case SERIAL_CONFIG_UART0_RX_UART0_TX:
        case SERIAL_CONFIG_UART1_RX_UART1_TX:
        {
            /* Check the Rx FIFO Empty bit */
            readable = ((obj->reg_base->UARTFR & (1 << 4)) != (1 << 4));
        }
        break;
        default:
        {
            MBED_ASSERT(false);
        }
        break;
    }

    return (int) readable;
}

int serial_writable(serial_t *obj)
{
    /* Check the "UART TX FIFO full" bit:
     * only if this is 0 can we transmit  */
    return (obj->reg_base->UARTFR & (1 << 5)) != (1 << 5);
}

void serial_break_set(serial_t *obj)
{
    /* Disable UART while writing to control registers */
    obj->reg_base->UARTCR &= ~(1 << 0);

    /* Set bit 1 of the line control register */
    obj->reg_base->UARTLCR_H |= 1 << 0;

    /* Enable the UART again */
    obj->reg_base->UARTCR |= 1 << 0;
}

void serial_break_clear(serial_t *obj)
{
    /* Disable UART while writing to control registers */
    obj->reg_base->UARTCR &= ~(1 << 0);

    /* Clear bit 1 of the line control register */
    obj->reg_base->UARTLCR_H &= ~(1 << 0);

    /* Enable the UART again */
    obj->reg_base->UARTCR |= 1 << 0;
}


text:
package org.elasticsearch.script;

import org.apache.logging.log4j.Logger;
import org.elasticsearch.ResourceNotFoundException;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.Diff;
import org.elasticsearch.cluster.DiffableUtils;
import org.elasticsearch.cluster.NamedDiff;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.logging.ESLoggerFactory;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;

import java.io.IOException;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Map;

/**
 * {@link ScriptMetaData} is used to store user-defined scripts
 * as part of the {@link ClusterState}.  Currently scripts can
 * be stored as part of the new namespace for a stored script where
 * only an id is used or as part of the deprecated namespace where
 * both a language and an id are used.
 */
public final class ScriptMetaData implements MetaData.Custom, Writeable, ToXContent {

    /**
     * A builder used to modify the currently stored scripts data held within
     * the {@link ClusterState}.  Scripts can be added or deleted, then built
     * to generate a new {@link Map} of scripts that will be used to update
     * the current {@link ClusterState}.
     */
    public static final class Builder {

        private final Map<String, StoredScriptSource> scripts;

        /**
         * @param previous The current {@link ScriptMetaData} or {@code null} if there
         *                 is no existing {@link ScriptMetaData}.
         */
        public Builder(ScriptMetaData previous) {
            this.scripts = previous == null ? new HashMap<>() :new HashMap<>(previous.scripts);
        }

        /**
         * Add a new script to the existing stored scripts.  The script will be added under
         * both the new namespace and the deprecated namespace, so that look ups under
         * the deprecated namespace will continue to work.  Should a script already exist under
         * the new namespace using a different language, it will be replaced and a deprecation
         * warning will be issued.  The replaced script will still exist under the deprecated
         * namespace and can continue to be looked up this way until it is deleted.
         *

         * Take for example script 'A' with lang 'L0' and data 'D0'.  If we add script 'A' to the
         * empty set, the scripts {@link Map} will be ["A" -- D0, "A#L0" -- D0].  If a script
         * 'A' with lang 'L1' and data 'D1' is then added, the scripts {@link Map} will be
         * ["A" -- D1, "A#L1" -- D1, "A#L0" -- D0].
         * @param id The user-specified id to use for the look up.
         * @param source The user-specified stored script data held in {@link StoredScriptSource}.
         */
        public Builder storeScript(String id, StoredScriptSource source) {
            StoredScriptSource previous = scripts.put(id, source);
            scripts.put(source.getLang() + "#" + id, source);

            if (previous != null && previous.getLang().equals(source.getLang()) == false) {
                DEPRECATION_LOGGER.deprecated("stored script [" + id + "] already exists using a different lang " +
                    "[" + previous.getLang() + "], the new namespace for stored scripts will only use (id) instead of (lang, id)");
            }

            return this;
        }

        /**
         * Delete a script from the existing stored scripts.  The script will be removed from the
         * new namespace if the script language matches the current script under the same id or
         * if the script language is {@code null}.  The script will be removed from the deprecated
         * namespace on any delete either using using the specified lang parameter or the language
         * found from looking up the script in the new namespace.
         *

         * Take for example a scripts {@link Map} with {"A" -- D1, "A#L1" -- D1, "A#L0" -- D0}.
         * If a script is removed specified by an id 'A' and lang {@code null} then the scripts
         * {@link Map} will be {"A#L0" -- D0}.  To remove the final script, the deprecated
         * namespace must be used, so an id 'A' and lang 'L0' would need to be specified.
         * @param id The user-specified id to use for the look up.
         * @param lang The user-specified language to use for the look up if using the deprecated
         *             namespace, otherwise {@code null}.
         */
        public Builder deleteScript(String id, String lang) {
            StoredScriptSource source = scripts.get(id);

            if (lang == null) {
                if (source == null) {
                    throw new ResourceNotFoundException("stored script [" + id + "] does not exist and cannot be deleted");
                }

                lang = source.getLang();
            }

            if (source != null) {
                if (lang.equals(source.getLang())) {
                    scripts.remove(id);
                }
            }

            source = scripts.get(lang + "#" + id);

            if (source == null) {
                throw new ResourceNotFoundException(
                    "stored script [" + id + "] using lang [" + lang + "] does not exist and cannot be deleted");
            }

            scripts.remove(lang + "#" + id);

            return this;
        }

        /**
         * @return A {@link ScriptMetaData} with the updated {@link Map} of scripts.
         */
        public ScriptMetaData build() {
            return new ScriptMetaData(scripts);
        }
    }

    static final class ScriptMetadataDiff implements NamedDiff<MetaData.Custom> {

        final Diff<Map<String, StoredScriptSource>> pipelines;

        ScriptMetadataDiff(ScriptMetaData before, ScriptMetaData after) {
            this.pipelines = DiffableUtils.diff(before.scripts, after.scripts, DiffableUtils.getStringKeySerializer());
        }

        ScriptMetadataDiff(StreamInput in) throws IOException {
            pipelines = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(),
                StoredScriptSource::new, StoredScriptSource::readDiffFrom);
        }

        @Override
        public String getWriteableName() {
            return TYPE;
        }

        @Override
        public MetaData.Custom apply(MetaData.Custom part) {
            return new ScriptMetaData(pipelines.apply(((ScriptMetaData) part).scripts));
        }

        @Override
        public void writeTo(StreamOutput out) throws IOException {
            pipelines.writeTo(out);
        }
    }

    /**
     * Convenience method to build and return a new
     * {@link ScriptMetaData} adding the specified stored script.
     */
    static ScriptMetaData putStoredScript(ScriptMetaData previous, String id, StoredScriptSource source) {
        Builder builder = new Builder(previous);
        builder.storeScript(id, source);

        return builder.build();
    }

    /**
     * Convenience method to build and return a new
     * {@link ScriptMetaData} deleting the specified stored script.
     */
    static ScriptMetaData deleteStoredScript(ScriptMetaData previous, String id, String lang) {
        Builder builder = new ScriptMetaData.Builder(previous);
        builder.deleteScript(id, lang);

        return builder.build();
    }

    /**
     * Standard logger necessary for allocation of the deprecation logger.
     */
    private static final Logger LOGGER = ESLoggerFactory.getLogger(ScriptMetaData.class);

    /**
     * Deprecation logger necessary for namespace changes related to stored scripts.
     */
    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LOGGER);

    /**
     * The type of {@link ClusterState} data.
     */
    public static final String TYPE = "stored_scripts";

    /**
     * This will parse XContent into {@link ScriptMetaData}.
     *
     * The following format will be parsed for the new namespace:
     *
     * {@code
     * {
     *     "<id>" : "<{@link StoredScriptSource#fromXContent(XContentParser)}>",
     *     "<id>" : "<{@link StoredScriptSource#fromXContent(XContentParser)}>",
     *     ...
     * }
     * }
     *
     * The following format will be parsed for the deprecated namespace:
     *
     * {@code
     * {
     *     "<id>" : "",
     *     "<id>" : "",
     *     ...
     * }
     * }
     *
     * Note when using the deprecated namespace, the language will be pulled from
     * the id and options will be set to an empty {@link Map}.
     */
    public static ScriptMetaData fromXContent(XContentParser parser) throws IOException {
        Map<String, StoredScriptSource> scripts = new HashMap<>();
        String id = null;
        StoredScriptSource source;

        Token token = parser.currentToken();

        if (token == null) {
            token = parser.nextToken();
        }

        if (token != Token.START_OBJECT) {
            throw new ParsingException(parser.getTokenLocation(), "unexpected token [" + token + "], expected [{]");
        }

        token = parser.nextToken();

        while (token != Token.END_OBJECT) {
            switch (token) {
                case FIELD_NAME:
                    id = parser.currentName();
                    break;
                case VALUE_STRING:
                    if (id == null) {
                        throw new ParsingException(parser.getTokenLocation(),
                            "unexpected token [" + token + "], expected [<id>, , {]");
                    }

                    int split = id.indexOf('#');

                    if (split == -1) {
                        throw new IllegalArgumentException("illegal stored script id [" + id + "], does not contain lang");
                    } else {
                        source = new StoredScriptSource(id.substring(0, split), parser.text(), Collections.emptyMap());
                    }
                    scripts.put(id, source);

                    id = null;

                    break;
                case START_OBJECT:
                    if (id == null) {
                        throw new ParsingException(parser.getTokenLocation(),
                            "unexpected token [" + token + "], expected [<id>, , {]");
                    }

                    source = StoredScriptSource.fromXContent(parser);
                    scripts.put(id, source);

                    break;
                default:
                    throw new ParsingException(parser.getTokenLocation(), "unexpected token [" + token + "], expected [<id>, , {]");
            }

            token = parser.nextToken();
        }

        return new ScriptMetaData(scripts);
    }

    public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
        return new ScriptMetadataDiff(in);
    }

    private final Map<String, StoredScriptSource> scripts;

    /**
     * Standard constructor to create metadata to store scripts.
     * @param scripts The currently stored scripts.  Must not be {@code null},
     *                use and empty {@link Map} to specify there were no
     *                previously stored scripts.
     */
    ScriptMetaData(Map<String, StoredScriptSource> scripts) {
        this.scripts = Collections.unmodifiableMap(scripts);
    }

    public ScriptMetaData(StreamInput in) throws IOException {
        Map<String, StoredScriptSource> scripts = new HashMap<>();
        StoredScriptSource source;
        int size = in.readVInt();

        for (int i = 0; i < size; i++) {
            String id = in.readString();

            // Prior to version 5.3 all scripts were stored using the deprecated namespace.
            // Split the id to find the language then use StoredScriptSource to parse the
            // expected BytesReference after which a new StoredScriptSource is created
            // with the appropriate language and options.
            if (in.getVersion().before(Version.V_5_3_0)) {
                int split = id.indexOf('#');

                if (split == -1) {
                    throw new IllegalArgumentException("illegal stored script id [" + id + "], does not contain lang");
                } else {
                    source = new StoredScriptSource(in);
                    source = new StoredScriptSource(id.substring(0, split), source.getCode(), Collections.emptyMap());
                }
            // Version 5.3+ can just be parsed normally using StoredScriptSource.
            } else {
                source = new StoredScriptSource(in);
            }

            scripts.put(id, source);
        }

        this.scripts = Collections.unmodifiableMap(scripts);
    }

    @Override
    public void writeTo(StreamOutput out) throws IOException {
        // Version 5.3+ will output the contents of the scripts' Map using
        // StoredScriptSource to stored the language, code, and options.
        if (out.getVersion().onOrAfter(Version.V_5_3_0)) {
            out.writeVInt(scripts.size());

            for (Map.Entry<String, StoredScriptSource> entry : scripts.entrySet()) {
                out.writeString(entry.getKey());
                entry.getValue().writeTo(out);
            }
        // Prior to Version 5.3, stored scripts can only be read using the deprecated
        // namespace.  Scripts using the deprecated namespace are first isolated in a
        // temporary Map, then written out.  Since all scripts will be stored using the
        // deprecated namespace, no scripts will be lost.
        } else {
            Map<String, StoredScriptSource> filtered = new HashMap<>();

            for (Map.Entry<String, StoredScriptSource> entry : scripts.entrySet()) {
                if (entry.getKey().contains("#")) {
                    filtered.put(entry.getKey(), entry.getValue());
                }
            }

            out.writeVInt(filtered.size());

            for (Map.Entry<String, StoredScriptSource> entry : filtered.entrySet()) {
                out.writeString(entry.getKey());
                entry.getValue().writeTo(out);
            }
        }
    }

    /**
     * This will write XContent from {@link ScriptMetaData}.  The following format will be written:
     *
     * {@code
     * {
     *     "<id>" : "<{@link StoredScriptSource#toXContent(XContentBuilder, Params)}>",
     *     "<id>" : "<{@link StoredScriptSource#toXContent(XContentBuilder, Params)}>",
     *     ...
     * }
     * }
     */
    @Override
    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
        for (Map.Entry<String, StoredScriptSource> entry : scripts.entrySet()) {
            builder.field(entry.getKey());
            entry.getValue().toXContent(builder, params);
        }

        return builder;
    }

    @Override
    public Diff<MetaData.Custom> diff(MetaData.Custom before) {
        return new ScriptMetadataDiff((ScriptMetaData)before, this);
    }

    @Override
    public String getWriteableName() {
        return TYPE;
    }

    @Override
    public EnumSet<MetaData.XContentContext> context() {
        return MetaData.ALL_CONTEXTS;
    }

    /**
     * Retrieves a stored script from the new namespace if lang is {@code null}.
     * Otherwise, returns a stored script from the deprecated namespace.  Either
     * way an id is required.
     */
    StoredScriptSource getStoredScript(String id, String lang) {
        if (lang == null) {
            return scripts.get(id);
        } else {
            return scripts.get(lang + "#" + id);
        }
    }

    @Override
    public boolean equals(Object o) {
        if (this == o) return true;
        if (o == null || getClass() != o.getClass()) return false;

        ScriptMetaData that = (ScriptMetaData)o;

        return scripts.equals(that.scripts);

    }

    @Override
    public int hashCode() {
        return scripts.hashCode();
    }

    @Override
    public String toString() {
        return "ScriptMetaData{" +
            "scripts=" + scripts +
            '}';
    }
}

redpajama_set_name: RedPajamaGithub


text: Miscellaneous stand-alone apps used for testing purposes.


text:
<!DOCTYPE html>

        <meta charset="utf-8">
        <meta http-equiv="X-UA-Compatible" content="IE=edge">
        <meta name="viewport" content="width=device-width, initial-scale=1.0">
        <meta name="author" content="Amit Kushwaha" />
        <meta name="copyright" content="Amit Kushwaha" />

        <meta name="twitter:creator" content="@imYardstick17">
        <meta property="og:type" content="article" />
        <meta name="twitter:card" content="summary">

<meta name="keywords" content=", Data Engineering, " />

<meta property="og:title" content="Example to use Scalar Pandas UDF and UDF "/>
<meta property="og:url" content="https://amitk.org/example-to-use-scalar-pandas-udf-and-udf.html" />
<meta property="og:description" content="This is a notebook to showcase usage of pandas udf" />
<meta property="og:site_name" content="Connecting Dots" />
<meta property="og:article:author" content="Amit Kushwaha" />
<meta property="og:article:published_time" content="2019-05-31T00:00:00+05:30" />
<meta name="twitter:title" content="Example to use Scalar Pandas UDF and UDF ">
<meta name="twitter:description" content="This is a notebook to showcase usage of pandas udf">

        <title>Example to use Scalar Pandas UDF and UDF  · Connecting Dots
</title>
        <link href="//netdna.bootstrapcdn.com/twitter-bootstrap/2.3.2/css/bootstrap-combined.min.css" rel="stylesheet">
        <link href="//netdna.bootstrapcdn.com/font-awesome/4.0.1/css/font-awesome.css" rel="stylesheet">
        <link rel="stylesheet" type="text/css" href="https://amitk.org/theme/css/pygments.css" media="screen">
        <link rel="stylesheet" type="text/css" href="https://amitk.org/theme/tipuesearch/tipuesearch.css" media="screen">
        <link rel="stylesheet" type="text/css" href="https://amitk.org/theme/css/elegant.css" media="screen">
        <link rel="stylesheet" type="text/css" href="https://amitk.org/theme/css/custom.css" media="screen">

                    Connecting Dots

Home

About me

Categories

Tags

Archives

<form class="navbar-search" action="https://amitk.org/search.html" onsubmit="return validateForm(this.elements['q'].value);"> <input type="text" class="search-query" placeholder="Search" name="q" id="tipue_search_input"></form>

 Example to use Scalar Pandas UDF and UDF

In [3]:

from pyspark.sql.functions import col, udf
from pyspark.sql.types import StructField, StringType, IntegerType, StructType
schema = StructType(
    [
        StructField("Name", StringType(), True),
        StructField("Age", IntegerType(), True),
        StructField("Gender", StringType(), True),
    ]
)

df = spark.createDataFrame([ ['Raju', 24, 'Male'], ['Sita', 26, 'Female'], ['Andrew', 25, 'Male'] ]* 1000000 , schema=schema)

In [4]:

df.show()

+------+---+------+
|  Name|Age|Gender|
+------+---+------+
|  Raju| 24|  Male|
|  Sita| 26|Female|
|Andrew| 25|  Male|
|  Raju| 24|  Male|
|  Sita| 26|Female|
|Andrew| 25|  Male|
|  Raju| 24|  Male|
|  Sita| 26|Female|
|Andrew| 25|  Male|
|  Raju| 24|  Male|
|  Sita| 26|Female|
|Andrew| 25|  Male|
|  Raju| 24|  Male|
|  Sita| 26|Female|
|Andrew| 25|  Male|
|  Raju| 24|  Male|
|  Sita| 26|Female|
|Andrew| 25|  Male|
|  Raju| 24|  Male|
|  Sita| 26|Female|
+------+---+------+
only showing top 20 rows

In [5]:

%%time

def square(x):
    return x**2

square_udf_float = udf(lambda z: square(z), IntegerType())

df.select('Name','Age', square_udf_float('Age').alias('Age_squared')).collect()

CPU times: user 10.7 s, sys: 520 ms, total: 11.2 s
Wall time: 21.2 s

In [6]:

%%time

from pyspark.sql.functions import PandasUDFType, pandas_udf

def square_ages(ages):
    return ages**2

@pandas_udf('integer', returnType=PandasUDFType.SCALAR)
def pandas_udf_square(ages):
    return square_ages(ages)

df.select('Name','Age', pandas_udf_square('Age').alias('Age_squared')).collect()

CPU times: user 10.6 s, sys: 420 ms, total: 11 s
Wall time: 20.9 s

                Comments

Please enable JavaScript to view the comments powered by Disqus.
comments powered by Disqus

Published

            <time pubdate="pubdate" datetime="2019-05-31T00:00:00+05:30">May 31, 2019</time>

Category

            Data Engineering

Contact

Powered by Pelican. Theme: Elegant by Talha Mansoor


text:
'use strict';

angular.module('footieApp')
  .controller('MainCtrl', function() {
});


text:
ACCEPTED

#### According to
Index Fungorum

#### Published in
Mycotaxon 102: 254 (2007)

#### Original name
Hansfordia pallens K.X. Hu & S.X. Guo

### Remarks
null


text:
from __future__ import unicode_literals

from django.contrib.gis.geos import HAS_GEOS
from django.contrib.gis.tests.utils import no_oracle
from django.db import connection
from django.test import TestCase, skipUnlessDBFeature
from django.test.utils import override_settings
from django.utils import timezone

if HAS_GEOS:
    from django.contrib.gis.db.models import Collect, Count, Extent, F, Union
    from django.contrib.gis.geometry.backend import Geometry
    from django.contrib.gis.geos import GEOSGeometry, Point, MultiPoint

    from .models import City, Location, DirectoryEntry, Parcel, Book, Author, Article, Event

@skipUnlessDBFeature("gis_enabled")
class RelatedGeoModelTest(TestCase):
    fixtures = ['initial']

    def test02_select_related(self):
        "Testing `select_related` on geographic models (see #7126)."
        qs1 = City.objects.order_by('id')
        qs2 = City.objects.order_by('id').select_related()
        qs3 = City.objects.order_by('id').select_related('location')

        # Reference data for what's in the fixtures.
        cities = (
            ('Aurora', 'TX', -97.516111, 33.058333),
            ('Roswell', 'NM', -104.528056, 33.387222),
            ('Kecksburg', 'PA', -79.460734, 40.18476),
        )

        for qs in (qs1, qs2, qs3):
            for ref, c in zip(cities, qs):
                nm, st, lon, lat = ref
                self.assertEqual(nm, c.name)
                self.assertEqual(st, c.state)
                self.assertEqual(Point(lon, lat), c.location.point)

    @skipUnlessDBFeature("has_transform_method")
    def test03_transform_related(self):
        "Testing the `transform` GeoQuerySet method on related geographic models."
        # All the transformations are to state plane coordinate systems using
        # US Survey Feet (thus a tolerance of 0 implies error w/in 1 survey foot).
        tol = 0

        def check_pnt(ref, pnt):
            self.assertAlmostEqual(ref.x, pnt.x, tol)
            self.assertAlmostEqual(ref.y, pnt.y, tol)
            self.assertEqual(ref.srid, pnt.srid)

        # Each city transformed to the SRID of their state plane coordinate system.
        transformed = (('Kecksburg', 2272, 'POINT(1490553.98959621 314792.131023984)'),
                       ('Roswell', 2257, 'POINT(481902.189077221 868477.766629735)'),
                       ('Aurora', 2276, 'POINT(2269923.2484839 7069381.28722222)'),
                       )

        for name, srid, wkt in transformed:
            # Doing this implicitly sets `select_related` select the location.
            # TODO: Fix why this breaks on Oracle.
            qs = list(City.objects.filter(name=name).transform(srid, field_name='location__point'))
            check_pnt(GEOSGeometry(wkt, srid), qs[0].location.point)

    @skipUnlessDBFeature("supports_extent_aggr")
    def test04a_related_extent_aggregate(self):
        "Testing the `extent` GeoQuerySet aggregates on related geographic models."
        # This combines the Extent and Union aggregates into one query
        aggs = City.objects.aggregate(Extent('location__point'))

        # One for all locations, one that excludes New Mexico (Roswell).
        all_extent = (-104.528056, 29.763374, -79.460734, 40.18476)
        txpa_extent = (-97.516111, 29.763374, -79.460734, 40.18476)
        e1 = City.objects.extent(field_name='location__point')
        e2 = City.objects.exclude(state='NM').extent(field_name='location__point')
        e3 = aggs['location__point__extent']

        # The tolerance value is to four decimal places because of differences
        # between the Oracle and PostGIS spatial backends on the extent calculation.
        tol = 4
        for ref, e in [(all_extent, e1), (txpa_extent, e2), (all_extent, e3)]:
            for ref_val, e_val in zip(ref, e):
                self.assertAlmostEqual(ref_val, e_val, tol)

    @skipUnlessDBFeature("has_unionagg_method")
    def test04b_related_union_aggregate(self):
        "Testing the `unionagg` GeoQuerySet aggregates on related geographic models."
        # This combines the Extent and Union aggregates into one query
        aggs = City.objects.aggregate(Union('location__point'))

        # These are the points that are components of the aggregate geographic
        # union that is returned.  Each point # corresponds to City PK.
        p1 = Point(-104.528056, 33.387222)
        p2 = Point(-97.516111, 33.058333)
        p3 = Point(-79.460734, 40.18476)
        p4 = Point(-96.801611, 32.782057)
        p5 = Point(-95.363151, 29.763374)

        # The second union aggregate is for a union
        # query that includes limiting information in the WHERE clause (in other
        # words a `.filter()` precedes the call to `.unionagg()`).
        ref_u1 = MultiPoint(p1, p2, p4, p5, p3, srid=4326)
        ref_u2 = MultiPoint(p2, p3, srid=4326)

        u1 = City.objects.unionagg(field_name='location__point')
        u2 = City.objects.exclude(
            name__in=('Roswell', 'Houston', 'Dallas', 'Fort Worth'),
        ).unionagg(field_name='location__point')
        u3 = aggs['location__point__union']
        self.assertEqual(type(u1), MultiPoint)
        self.assertEqual(type(u3), MultiPoint)

        # Ordering of points in the result of the union is not defined and
        # implementation-dependent (DB backend, GEOS version)
        self.assertSetEqual({p.ewkt for p in ref_u1}, {p.ewkt for p in u1})
        self.assertSetEqual({p.ewkt for p in ref_u2}, {p.ewkt for p in u2})
        self.assertSetEqual({p.ewkt for p in ref_u1}, {p.ewkt for p in u3})

    def test05_select_related_fk_to_subclass(self):
        "Testing that calling select_related on a query over a model with an FK to a model subclass works"
        # Regression test for #9752.
        list(DirectoryEntry.objects.all().select_related())

    def test06_f_expressions(self):
        "Testing F() expressions on GeometryFields."
        # Constructing a dummy parcel border and getting the City instance for
        # assigning the FK.
        b1 = GEOSGeometry(
            'POLYGON((-97.501205 33.052520,-97.501205 33.052576,'
            '-97.501150 33.052576,-97.501150 33.052520,-97.501205 33.052520))',
            srid=4326
        )
        pcity = City.objects.get(name='Aurora')

        # First parcel has incorrect center point that is equal to the City;
        # it also has a second border that is different from the first as a
        # 100ft buffer around the City.
        c1 = pcity.location.point
        c2 = c1.transform(2276, clone=True)
        b2 = c2.buffer(100)
        Parcel.objects.create(name='P1', city=pcity, center1=c1, center2=c2, border1=b1, border2=b2)

        # Now creating a second Parcel where the borders are the same, just
        # in different coordinate systems.  The center points are also the
        # same (but in different coordinate systems), and this time they
        # actually correspond to the centroid of the border.
        c1 = b1.centroid
        c2 = c1.transform(2276, clone=True)
        Parcel.objects.create(name='P2', city=pcity, center1=c1, center2=c2, border1=b1, border2=b1)

        # Should return the second Parcel, which has the center within the
        # border.
        qs = Parcel.objects.filter(center1__within=F('border1'))
        self.assertEqual(1, len(qs))
        self.assertEqual('P2', qs[0].name)

        if connection.features.supports_transform:
            # This time center2 is in a different coordinate system and needs
            # to be wrapped in transformation SQL.
            qs = Parcel.objects.filter(center2__within=F('border1'))
            self.assertEqual(1, len(qs))
            self.assertEqual('P2', qs[0].name)

        # Should return the first Parcel, which has the center point equal
        # to the point in the City ForeignKey.
        qs = Parcel.objects.filter(center1=F('city__location__point'))
        self.assertEqual(1, len(qs))
        self.assertEqual('P1', qs[0].name)

        if connection.features.supports_transform:
            # This time the city column should be wrapped in transformation SQL.
            qs = Parcel.objects.filter(border2__contains=F('city__location__point'))
            self.assertEqual(1, len(qs))
            self.assertEqual('P1', qs[0].name)

    def test07_values(self):
        "Testing values() and values_list() and GeoQuerySets."
        # GeoQuerySet and GeoValuesQuerySet, and GeoValuesListQuerySet respectively.
        gqs = Location.objects.all()
        gvqs = Location.objects.values()
        gvlqs = Location.objects.values_list()

        # Incrementing through each of the models, dictionaries, and tuples
        # returned by the different types of GeoQuerySets.
        for m, d, t in zip(gqs, gvqs, gvlqs):
            # The values should be Geometry objects and not raw strings returned
            # by the spatial database.
            self.assertIsInstance(d['point'], Geometry)
            self.assertIsInstance(t[1], Geometry)
            self.assertEqual(m.point, d['point'])
            self.assertEqual(m.point, t[1])

    @override_settings(USE_TZ=True)
    def test_07b_values(self):
        "Testing values() and values_list() with aware datetime. See #21565."
        Event.objects.create(name="foo", when=timezone.now())
        list(Event.objects.values_list('when'))

    def test08_defer_only(self):
        "Testing defer() and only() on Geographic models."
        qs = Location.objects.all()
        def_qs = Location.objects.defer('point')
        for loc, def_loc in zip(qs, def_qs):
            self.assertEqual(loc.point, def_loc.point)

    def test09_pk_relations(self):
        "Ensuring correct primary key column is selected across relations. See #10757."
        # The expected ID values -- notice the last two location IDs
        # are out of order.  Dallas and Houston have location IDs that differ
        # from their PKs -- this is done to ensure that the related location
        # ID column is selected instead of ID column for the city.
        city_ids = (1, 2, 3, 4, 5)
        loc_ids = (1, 2, 3, 5, 4)
        ids_qs = City.objects.order_by('id').values('id', 'location__id')
        for val_dict, c_id, l_id in zip(ids_qs, city_ids, loc_ids):
            self.assertEqual(val_dict['id'], c_id)
            self.assertEqual(val_dict['location__id'], l_id)

    # TODO: fix on Oracle -- qs2 returns an empty result for an unknown reason
    @no_oracle
    def test10_combine(self):
        "Testing the combination of two GeoQuerySets.  See #10807."
        buf1 = City.objects.get(name='Aurora').location.point.buffer(0.1)
        buf2 = City.objects.get(name='Kecksburg').location.point.buffer(0.1)
        qs1 = City.objects.filter(location__point__within=buf1)
        qs2 = City.objects.filter(location__point__within=buf2)
        combined = qs1 | qs2
        names = [c.name for c in combined]
        self.assertEqual(2, len(names))
        self.assertIn('Aurora', names)
        self.assertIn('Kecksburg', names)

    # TODO: fix on Oracle -- get the following error because the SQL is ordered
    # by a geometry object, which Oracle apparently doesn't like:
    #  ORA-22901: cannot compare nested table or VARRAY or LOB attributes of an object type
    @no_oracle
    def test12a_count(self):
        "Testing `Count` aggregate use with the `GeoManager` on geo-fields."
        # The City, 'Fort Worth' uses the same location as Dallas.
        dallas = City.objects.get(name='Dallas')

        # Count annotation should be 2 for the Dallas location now.
        loc = Location.objects.annotate(num_cities=Count('city')).get(id=dallas.location.id)
        self.assertEqual(2, loc.num_cities)

    def test12b_count(self):
        "Testing `Count` aggregate use with the `GeoManager` on non geo-fields. See #11087."
        # Should only be one author (Trevor Paglen) returned by this query, and
        # the annotation should have 3 for the number of books, see #11087.
        # Also testing with a `GeoValuesQuerySet`, see #11489.
        qs = Author.objects.annotate(num_books=Count('books')).filter(num_books__gt=1)
        vqs = Author.objects.values('name').annotate(num_books=Count('books')).filter(num_books__gt=1)
        self.assertEqual(1, len(qs))
        self.assertEqual(3, qs[0].num_books)
        self.assertEqual(1, len(vqs))
        self.assertEqual(3, vqs[0]['num_books'])

    # TODO: fix on Oracle -- get the following error because the SQL is ordered
    # by a geometry object, which Oracle apparently doesn't like:
    #  ORA-22901: cannot compare nested table or VARRAY or LOB attributes of an object type
    @no_oracle
    def test13c_count(self):
        "Testing `Count` aggregate with `.values()`.  See #15305."
        qs = Location.objects.filter(id=5).annotate(num_cities=Count('city')).values('id', 'point', 'num_cities')
        self.assertEqual(1, len(qs))
        self.assertEqual(2, qs[0]['num_cities'])
        self.assertIsInstance(qs[0]['point'], GEOSGeometry)

    # TODO: The phantom model does appear on Oracle.
    @no_oracle
    def test13_select_related_null_fk(self):
        "Testing `select_related` on a nullable ForeignKey via `GeoManager`. See #11381."
        Book.objects.create(title='Without Author')
        b = Book.objects.select_related('author').get(title='Without Author')
        # Should be `None`, and not a 'dummy' model.
        self.assertEqual(None, b.author)

    @skipUnlessDBFeature("supports_collect_aggr")
    def test14_collect(self):
        "Testing the `collect` GeoQuerySet method and `Collect` aggregate."
        # Reference query:
        # SELECT AsText(ST_Collect("relatedapp_location"."point")) FROM "relatedapp_city" LEFT OUTER JOIN
        #    "relatedapp_location" ON ("relatedapp_city"."location_id" = "relatedapp_location"."id")
        #    WHERE "relatedapp_city"."state" = 'TX';
        ref_geom = GEOSGeometry(
            'MULTIPOINT(-97.516111 33.058333,-96.801611 32.782057,'
            '-95.363151 29.763374,-96.801611 32.782057)'
        )

        c1 = City.objects.filter(state='TX').collect(field_name='location__point')
        c2 = City.objects.filter(state='TX').aggregate(Collect('location__point'))['location__point__collect']

        for coll in (c1, c2):
            # Even though Dallas and Ft. Worth share same point, Collect doesn't
            # consolidate -- that's why 4 points in MultiPoint.
            self.assertEqual(4, len(coll))
            self.assertTrue(ref_geom.equals(coll))

    def test15_invalid_select_related(self):
        "Testing doing select_related on the related name manager of a unique FK. See #13934."
        qs = Article.objects.select_related('author__article')
        # This triggers TypeError when `get_default_columns` has no `local_only`
        # keyword.  The TypeError is swallowed if QuerySet is actually
        # evaluated as list generation swallows TypeError in CPython.
        str(qs.query)

    def test16_annotated_date_queryset(self):
        "Ensure annotated date querysets work if spatial backend is used.  See #14648."
        birth_years = [dt.year for dt in
                       list(Author.objects.annotate(num_books=Count('books')).dates('dob', 'year'))]
        birth_years.sort()
        self.assertEqual([1950, 1974], birth_years)

    # TODO: Related tests for KML, GML, and distance lookups.


text:
var popmotion = require('./module.js'),
    UIref = window.ui;

window.ui = window.popmotion = popmotion;

/*
    If noConflict is run, the original reference to window.UI is
    restored and Popmotion is loaded to window.Popmotion
*/
window.ui.noConflict = function () {
    window.ui = UIref;
};