text:
/**@file
 *
 * @defgroup ble_sdk_srv_dfu Device Firmware Update Service
 * @{
 * @ingroup  ble_sdk_srv
 * @brief    Device Firmware Update Service
 *
 * @details  The Device Firmware Update (DFU) service is a GATT based service that can be used for
 *           performing firmware updates over BLE. Note that this implementation uses vendor
 *           specific UUIDs for service and characteristics and is intended to demonstrate the
 *           firmware updates over BLE. Refer @ref bledfu_transport_bleservice and @ref
 *            bledfu_transport_bleprofile for more information on the service and profile respectively.
 */

#ifndef BLE_DFU_H__
#define BLE_DFU_H__

#include <stdint.h>
#include "ble_gatts.h"
#include "ble_gap.h"
#include "ble.h"
#include "ble_srv_common.h"

#define BLE_DFU_SERVICE_UUID                 0x1530                       /**< The UUID of the DFU Service. */
#define BLE_DFU_PKT_CHAR_UUID                0x1532                       /**< The UUID of the DFU Packet Characteristic. */
#define BLE_DFU_CTRL_PT_UUID                 0x1531                       /**< The UUID of the DFU Control Point. */
#define BLE_DFU_STATUS_REP_UUID              0x1533                       /**< The UUID of the DFU Status Report Characteristic. */
#define BLE_DFU_REV_CHAR_UUID                0x1534                       /**< The UUID of the DFU Revision Characteristic. */

/**@brief   DFU Event type.
 *
 * @details This enumeration contains the types of events that will be received from the DFU Service.
 */
typedef enum
{
    BLE_DFU_START,                                                      /**< The event indicating that the peer wants the application to prepare for a new firmware update. */
    BLE_DFU_RECEIVE_INIT_DATA,                                          /**< The event indicating that the peer wants the application to prepare to receive init parameters. */
    BLE_DFU_RECEIVE_APP_DATA,                                           /**< The event indicating that the peer wants the application to prepare to receive the new firmware image. */
    BLE_DFU_VALIDATE,                                                   /**< The event indicating that the peer wants the application to validate the newly received firmware image. */
    BLE_DFU_ACTIVATE_N_RESET,                                           /**< The event indicating that the peer wants the application to undergo activate new firmware and restart with new valid application */
    BLE_DFU_SYS_RESET,                                                  /**< The event indicating that the peer wants the application to undergo a reset and start the currently valid application image.*/
    BLE_DFU_PKT_RCPT_NOTIF_ENABLED,                                     /**< The event indicating that the peer has enabled packet receipt notifications. It is the responsibility of the application to call @ref ble_dfu_pkts_rcpt_notify each time the number of packets indicated by num_of_pkts field in @ref ble_dfu_evt_t is received.*/
    BLE_DFU_PKT_RCPT_NOTIF_DISABLED,                                    /**< The event indicating that the peer has disabled the packet receipt notifications.*/
    BLE_DFU_PACKET_WRITE,                                               /**< The event indicating that the peer has written a value to the 'DFU Packet' characteristic. The data received from the peer will be present in the @ref BLE_DFU_PACKET_WRITE element contained within @ref ble_dfu_evt_t.*/
    BLE_DFU_BYTES_RECEIVED_SEND                                         /**< The event indicating that the peer is requesting for the number of bytes of firmware data last received by the application. It is the responsibility of the application to call @ref ble_dfu_pkts_rcpt_notify in response to this event. */
} ble_dfu_evt_type_t;

/**@brief   DFU Procedure type.
 *
 * @details This enumeration contains the types of DFU procedures.
 */
typedef enum
{
    BLE_DFU_START_PROCEDURE        = 1,                                 /**< DFU Start procedure.*/
    BLE_DFU_INIT_PROCEDURE         = 2,                                 /**< DFU Initialization procedure.*/
    BLE_DFU_RECEIVE_APP_PROCEDURE  = 3,                                 /**< Firmware receiving procedure.*/
    BLE_DFU_VALIDATE_PROCEDURE     = 4,                                 /**< Firmware image validation procedure .*/
    BLE_DFU_PKT_RCPT_REQ_PROCEDURE = 8                                  /**< Packet receipt notification request procedure. */
} ble_dfu_procedure_t;

/**@brief   DFU Response value type.
 */
typedef enum
{
    BLE_DFU_RESP_VAL_SUCCESS = 1,                                       /**< Success.*/
    BLE_DFU_RESP_VAL_INVALID_STATE,                                     /**< Invalid state.*/
    BLE_DFU_RESP_VAL_NOT_SUPPORTED,                                     /**< Operation not supported.*/
    BLE_DFU_RESP_VAL_DATA_SIZE,                                         /**< Data size exceeds limit.*/
    BLE_DFU_RESP_VAL_CRC_ERROR,                                         /**< CRC Error.*/
    BLE_DFU_RESP_VAL_OPER_FAILED                                        /**< Operation failed.*/
} ble_dfu_resp_val_t;

/**@brief   DFU Packet structure.
 *
 * @details This structure contains the value of the DFU Packet characteristic as written by the
 *          peer and the length of the value written. It will be filled by the DFU Service when the
 *          peer writes to the DFU Packet characteristic.
 */
typedef struct
{
    uint8_t *                    p_data;                                /**< Pointer to the received packet. This will point to a word aligned memory location.*/
    uint8_t                      len;                                   /**< Length of the packet received. */
} ble_dfu_pkt_write_t;

/**@brief   Packet receipt notification request structure.
 *
 * @details This structure contains the contents of the packet receipt notification request
 *          sent by the DFU Controller.
 */
typedef struct
{
    uint16_t                     num_of_pkts;                           /**< The number of packets of firmware data to be received by application before sending the next Packet Receipt Notification to the peer. */
} ble_pkt_rcpt_notif_req_t;

/**@brief   DFU Event structure.
 *
 * @details This structure contains the event generated by the DFU Service based on the data
 *          received from the peer.
 */
typedef struct
{
    ble_dfu_evt_type_t           ble_dfu_evt_type;                      /**< Type of the event.*/
    union
    {
        ble_dfu_pkt_write_t      ble_dfu_pkt_write;                     /**< The DFU packet received. This field is when the @ref ble_dfu_evt_type field is set to @ref BLE_DFU_PACKET_WRITE.*/
        ble_pkt_rcpt_notif_req_t pkt_rcpt_notif_req;                    /**< Packet receipt notification request. This field is when the @ref ble_dfu_evt_type field is set to @ref BLE_DFU_PKT_RCPT_NOTIF_ENABLED.*/
    } evt;
} ble_dfu_evt_t;

// Forward declaration of the ble_dfu_t type.
typedef struct ble_dfu_s ble_dfu_t;

/**@brief DFU Service event handler type. */
typedef void (*ble_dfu_evt_handler_t) (ble_dfu_t * p_dfu, ble_dfu_evt_t * p_evt);

/**@brief   DFU service structure.
 *
 * @details This structure contains status information related to the service.
 */
struct ble_dfu_s
{
    uint16_t                     conn_handle;                           /**< Handle of the current connection (as provided by the S110 SoftDevice). This will be BLE_CONN_HANDLE_INVALID when not in a connection. */
    uint16_t                     revision;                              /**< Handle of DFU Service (as provided by the S110 SoftDevice). */
    uint16_t                     service_handle;                        /**< Handle of DFU Service (as provided by the S110 SoftDevice). */
    uint8_t                      uuid_type;                             /**< UUID type assigned for DFU Service by the S110 SoftDevice. */
    ble_gatts_char_handles_t     dfu_pkt_handles;                       /**< Handles related to the DFU Packet characteristic. */
    ble_gatts_char_handles_t     dfu_ctrl_pt_handles;                   /**< Handles related to the DFU Control Point characteristic. */
    ble_gatts_char_handles_t     dfu_status_rep_handles;                /**< Handles related to the DFU Status Report characteristic. */
    ble_gatts_char_handles_t     dfu_rev_handles;                       /**< Handles related to the DFU Revision characteristic. */
    ble_dfu_evt_handler_t        evt_handler;                           /**< The event handler to be called when an event is to be sent to the application.*/
    ble_srv_error_handler_t      error_handler;                         /**< Function to be called in case of an error. */
};

/**@brief      DFU service initialization structure.
 *
 * @details    This structure contains the initialization information for the DFU Service. The
 *             application needs to fill this structure and pass it to the DFU Service using the
 *             @ref ble_dfu_init function.
 */
typedef struct
{
    uint16_t                     revision;                              /**< Revision number to be exposed by the DFU service. */
    ble_dfu_evt_handler_t        evt_handler;                           /**< Event handler to be called for handling events in the Device Firmware Update Service. */
    ble_srv_error_handler_t      error_handler;                         /**< Function to be called in case of an error. */
} ble_dfu_init_t;

/**@brief      Function for handling a BLE event.
 *
 * @details    The DFU service expects the application to call this function each time an event
 *             is received from the S110 SoftDevice. This function processes the event, if it is
 *             relevant for the DFU service and calls the DFU event handler of the application if
 *             necessary.
 *
 * @param[in]  p_dfu        Pointer to the DFU service structure.
 * @param[in]  p_ble_evt    Pointer to the event received from S110 SoftDevice.
 */
void ble_dfu_on_ble_evt(ble_dfu_t * p_dfu, ble_evt_t * p_ble_evt);

/**@brief      Function for initializing the DFU service.
 *
 * @param[out] p_dfu        Device Firmware Update service structure. This structure will have to be
 *                          supplied by the application. It will be initialized by this function,
 *                          and will later be used to identify the service instance.
 * @param[in]  p_dfu_init   Information needed to initialize the service.
 *
 * @return     NRF_SUCCESS if the DFU service and its characteristics were successfully added to the
 *             S110 SoftDevice. Otherwise an error code.
 *             This function returns NRF_ERROR_NULL if the value of evt_handler in p_dfu_init
 *             structure provided is NULL or if the pointers supplied as input are NULL.
 */
uint32_t ble_dfu_init(ble_dfu_t * p_dfu, ble_dfu_init_t * p_dfu_init);

/**@brief       Function for sending response to a control point command.
 *
 * @details     This function will encode a DFU Control Point response using the given input
 *              parameters and will send a notification of the same to the peer.
 *
 * @param[in]   p_dfu       Pointer to the DFU service structure.
 * @param[in]   dfu_proc    Procedure for which this response is to be sent.
 * @param[in]   resp_val    Response value.
 *
 * @return      NRF_SUCCESS if the DFU Service has successfully requested the S110 SoftDevice to
 *              send the notification. Otherwise an error code.
 *              This function returns NRF_ERROR_INVALID_STATE if the device is not connected to a
 *              peer or if the DFU service is not initialized or if the notification of the DFU
 *              Status Report characteristic was not enabled by the peer. It returns NRF_ERROR_NULL
 *              if the pointer p_dfu is NULL.
 */
uint32_t ble_dfu_response_send(ble_dfu_t *          p_dfu,
                               ble_dfu_procedure_t  dfu_proc,
                               ble_dfu_resp_val_t   resp_val);

/**@brief      Function for notifying the peer about the number of bytes of firmware data received.
 *
 * @param[in]  p_dfu                      Pointer to the DFU service structure.
 * @param[in]  num_of_firmware_bytes_rcvd Number of bytes.
 *
 * @return     NRF_SUCCESS if the DFU Service has successfully requested the S110 SoftDevice to send
 *             the notification. Otherwise an error code.
 *             This function returns NRF_ERROR_INVALID_STATE if the device is not connected to a
 *             peer or if the DFU service is not initialized or if the notification of the DFU
 *             Status Report characteristic was not enabled by the peer. It returns NRF_ERROR_NULL
 *             if the pointer p_dfu is NULL.
 */
uint32_t ble_dfu_bytes_rcvd_report(ble_dfu_t * p_dfu, uint32_t num_of_firmware_bytes_rcvd);

/**@brief      Function for sending Packet Receipt Notification to the peer.
 *
 *             This function will encode the number of bytes received as input parameter into a
 *             notification of the control point characteristic and send it to the peer.
 *
 * @param[in]  p_dfu                      Pointer to the DFU service structure.
 * @param[in]  num_of_firmware_bytes_rcvd Number of bytes of firmware image received.
 *
 * @return     NRF_SUCCESS if the DFU Service has successfully requested the S110 SoftDevice to send
 *             the notification. Otherwise an error code.
 *             This function returns NRF_ERROR_INVALID_STATE if the device is not connected to a
 *             peer or if the DFU service is not initialized or if the notification of the DFU
 *             Status Report characteristic was not enabled by the peer. It returns NRF_ERROR_NULL
 *             if the pointer p_dfu is NULL.
 */
uint32_t ble_dfu_pkts_rcpt_notify(ble_dfu_t * p_dfu, uint32_t num_of_firmware_bytes_rcvd);

#endif // BLE_DFU_H__

/** @} */

redpajama_set_name: RedPajamaGithub


text:
England Championship 21/22
15/01/22 Championship 21/22 Barnsley Blackpool 30% 33% 38% 35% 65%
15/01/22 Championship 21/22 Derby County Sheffield United 38% 34% 29% 48% 52% 2 - 0
15/01/22 Championship 21/22 Huddersfield Town Swansea City 57% 18% 26% 54% 46% 1 - 1
15/01/22 Championship 21/22 Luton Town AFC Bournemouth 26% 34% 41% 42% 58% 3 - 2
15/01/22 Championship 21/22 Millwall FC Nottingham Forest 35% 35% 31% 41% 59% 0 - 1
15/01/22 Championship 21/22 Cardiff City Blackburn Rovers 23% 23% 55% 45% 55% 0 - 1
15/01/22 Championship 21/22 Peterborough United Coventry City 36% 41% 24% 43% 57% 1 - 4
15/01/22 Championship 21/22 Preston North End Birmingham City 48% 31% 22% 45% 55% 1 - 1
15/01/22 Championship 21/22 Middlesbrough FC Reading FC 59% 8% 34% 43% 57% 2 - 1
15/01/22 Championship 21/22 Fulham FC Bristol City 57% 17% 27% 58% 42% 6 - 2
12/01/22 Championship 21/22 Barnsley Stoke City 31% 26% 44% 29% 71%
11/01/22 Championship 21/22 Reading FC Fulham FC 26% 30% 45% 48% 52% 0 - 7
03/01/22 Championship 21/22 Swansea City Fulham FC 27% 32% 42% 56% 44%
03/01/22 Championship 21/22 Nottingham Forest Barnsley 49% 28% 23% 47% 53%
03/01/22 Championship 21/22 Reading FC Derby County 39% 35% 27% 33% 67% 2 - 2
03/01/22 Championship 21/22 AFC Bournemouth Peterborough United 75% 13% 13% 61% 39%
03/01/22 Championship 21/22 Stoke City Preston North End 53% 26% 22% 46% 54% 1 - 2
02/01/22 Championship 21/22 Birmingham City Queens Park Rangers 36% 23% 42% 61% 39% 1 - 2
02/01/22 Championship 21/22 Bristol City Millwall FC 28% 40% 32% 45% 55% 3 - 2
02/01/22 Championship 21/22 Blackburn Rovers Huddersfield Town 54% 23% 24% 57% 43% 0 - 0
02/01/22 Championship 21/22 West Bromwich Albion Cardiff City 53% 31% 17% 55% 45% 1 - 1
01/01/22 Championship 21/22 Sheffield United Middlesbrough FC 37% 27% 37% 40% 60%
01/01/22 Championship 21/22 Coventry City Luton Town 47% 27% 27% 50% 50%
01/01/22 Championship 21/22 Blackpool Hull City 48% 16% 36% 38% 62% 1 - 0
30/12/21 Championship 21/22 AFC Bournemouth Cardiff City 46% 27% 28% 72% 28% 3 - 0
30/12/21 Championship 21/22 Bristol City Queens Park Rangers 33% 31% 37% 51% 49% 1 - 2
30/12/21 Championship 21/22 Nottingham Forest Huddersfield Town 38% 29% 34% 50% 50% 0 - 1
30/12/21 Championship 21/22 West Bromwich Albion Preston North End 60% 32% 9% 45% 55%
30/12/21 Championship 21/22 Stoke City Derby County 50% 36% 14% 32% 68% 1 - 2
29/12/21 Championship 21/22 Swansea City Luton Town 36% 36% 28% 52% 48%
29/12/21 Championship 21/22 Coventry City Millwall FC 50% 32% 19% 57% 43% 0 - 1
29/12/21 Championship 21/22 Birmingham City Peterborough United 64% 14% 23% 52% 48%
29/12/21 Championship 21/22 Sheffield United Hull City 49% 19% 33% 35% 65%
29/12/21 Championship 21/22 Blackburn Rovers Barnsley 66% 27% 8% 60% 40% 2 - 1
29/12/21 Championship 21/22 Blackpool Middlesbrough FC 39% 27% 35% 39% 61% 1 - 2
29/12/21 Championship 21/22 Reading FC Fulham FC 27% 27% 46% 44% 56%
27/12/21 Championship 21/22 Queens Park Rangers AFC Bournemouth 36% 30% 34% 38% 62% 0 - 1
27/12/21 Championship 21/22 Derby County West Bromwich Albion 35% 39% 27% 45% 55% 1 - 0
26/12/21 Championship 21/22 Luton Town Bristol City 46% 23% 32% 58% 42%
26/12/21 Championship 21/22 Middlesbrough FC Nottingham Forest 34% 27% 40% 42% 58% 2 - 0
26/12/21 Championship 21/22 Peterborough United Reading FC 50% 18% 32% 48% 52%
26/12/21 Championship 21/22 Preston North End Sheffield United 39% 31% 31% 50% 50%
26/12/21 Championship 21/22 Fulham FC Birmingham City 54% 25% 21% 44% 56%
26/12/21 Championship 21/22 Cardiff City Coventry City 27% 27% 46% 43% 57%
26/12/21 Championship 21/22 Huddersfield Town Blackpool 46% 27% 28% 40% 60% 3 - 2

redpajama_set_name: RedPajamaCommonCrawl


text:
Arabs in Aspic — Victims of Your Father's Agony
(Black Widow BWRCD 185-2 / BWR 185, 2015, CD / LP)
by Henry Schneider, Published 2015-12-09
Norwegian band Arabs in Aspic released their sixth album this past spring, an effort taking almost two years. Victim of Your Father's Agony presents more 70s-influenced hard prog rock and a welcome companion to their previous album Pictures in a Dream. And the new album sounds a bit more diverse. The opening song "You Can Prove Them Wrong" is replete with heavy Hammond organ riffs, hard rock guitar solos, and gratifying vocal harmonies. On the second tune, "Sad without You," the band is a bit experimental with a reversed track intro to a pseudo-pop ballad in the style of The Beatles. Then they kick things into high gear with "One," an organ and guitar dominated hard rock song with Mellotron and twinned guitar lines, similar to a combination of Wishbone Ash and Deep Purple. Track four, "The Turk and the Italian Restaurant," is a funky bass and organ instrumental with what sounds like Moog synth lines, making me think of Edgar Winter's "Frankenstein." The next track, "God Requires Insanity," could easily be a lost Black Sabbath track. "TV 3" is a short new wave/hard rock hybrid. Then there is the entertaining "Flight of the Halibut" with its driving bass line, guitar, and rousing Mellotron riffs. Track eight, "Saint-Palais-Sur-Mer Pt. 2," goes in a new direction with beautiful acoustic guitar and synth, drawing comparisons in my mind to Amazing Blondel. The album closes with the excellent title track, Mellotron and twinned guitar lines. Overall, quite a satisfying listening experience, and though Victim of Your Father's Agony is not a major step forward, it is obvious that the band is evolving and maturing.
Related artist(s): Arabs in Aspic
http://www.facebook.com/arabsinaspic
http://www.twitter.com/arabsinaspic
http://www.soundcloud.com/arabsinaspic
http://www.youtube.com/dragedraug


Ben Carson: We are all Israelis!
All the candidates for the next president of United States are competing with each other for Netanyahu and Jewish Lobby favor. However, the retired neurosurgeon Dr. Benjamin Solomon Carson, 63, who is seeking GOP nomination, do has edge over the rest of the AZZ-suckers. He, like Barack Obama is not only Black but also claims his African Jewish ancestry.
He also told Wallace to go to Israel and ask any Israeli and he would tell you how much he hate Barack Obama. I'm sure Carson know that Israelis hate Blacks and that include Carson and Obama.
In March, another Israeli millionaire poodle, Glenn Beck, declared: "It's over, Ben Carson ruined any chance at Presidency". He was commenting on Ben Carson's antisemitic statement at CNN while interviewed by Chris Cuomo. "Homosexuality is a choice, because a lot of people who go into prison, go in straight – and when they come out they're gay," Carson said.
However, when Carson rabbi reminded him that Homosexuality = Judaism, he apologized for making such a stupid statement.
Carson visited Holy Wailing Wall in Occupied East Jerusalem in December 2014 before jumping into the presidential race. This is mandatory for every American who wish to live in the White House.
Ben Carson's pro-Jewish stands include; Hamas and Hizbullah are terrorists organizations and must be destroyed; Iran hate Jewish people, and therefore, must be punished like Iraq and Libya; Palestinian should resettled in Egyptian Sinai; illegal immigrants are no threat to US economy; Obamacare is really, really bad; Washington should increase military aid to Israel, and Holocaust denying should be declared a hate crime.
Ben Carson's anti-Jewish stands include; anti-LGBT and anti-gun control; says Climate Change is a hoax, and no legalization of marijuana.
This entry was posted in World News and tagged Zionism. Bookmark the permalink.
Have you ever thought that maybe, maybe ….. the American Jews controlled almost every pro-Black (NAACP, ACLU, etc.) and anti-Black (KKK) movement to whitewash their guilt of playing a major role in African slavery!!
The implied notion reported in the Guardian that the primacy of Jewish suffering (which has long ago reached its conclusion) has been reconfigured to the genes of the children of Holocaust survivors, must mean – If the claims are to be believed – that the gene pool of the Palestinian's, who continue to suffer at the hands of their tormentors, must be overflowing.
To draw such a parallel would naturally mean that people like me are entitled to be smeared and defamed. Yippee, that means I can now join my brothers who get labelled as being antisemitic merely for criticising the illegal actions of the terror state of Israel.
No Muslim leader has denied the sufferings of Israelite people at the hands of their Christian tormentors in Europe in the past. Some, like Ahmadinejad, only question the unproven estimate of the holy Six Million death. Even the Auschwitz Museum has lowered its original estimate of 4 million to 2.5 million since 1999.
Dr. Richard Falk, former UNHRC special envoy for PA Territories has claimed that the so-called 'antisemitism' didn't exit in the Muslim world until the Zionist Jews occupied Palestine.
Rehmat, I agree. Just to add, Ahmadinejad didn't deny the Holocaust as the media claimed, nor did he claim that Israel should be wiped off the map. It would be rather odd for him to be an antisemite since Iran has the highest population of Jews in the middle east outside of Israel, But the media don't want the public informed of these facts.


text:
Clamps Wonder Dog Clamp Auto Adjust Toggle clamps Cabinetry Clamp The BESSEY Auto-Adjust Toggle Clamps automatically adjust to variations in work piece dimensions while maintaining constant clamping force. This clamping force can also be easily adjusted making these clamps more versatile than ever. The cabinetry clamp is designed to combine the multiple steps of face frame cabinet installation into one easy process. Face frame stiles can be tightened, aligned, pre-drilled and fastened all while the clamp is in place. The clamp holds the individual cabinet edges together while also keeping the faces along the joint flush. There is an integrated pilot-hole drilling guide on one end. This guide flips out of the way so you can drive screws to connect the cabinets without removing or repositioning the clamp. accomplished by rubbing the scratch with the wax stick until filled. The final step is to level off the repair excess material with the built in leveler. The repair, color replacement and filling, all done with one product. BESSEY CABINETRY CLAMP / FACE FRAMES AC85110 ......................................................$26.95 13/16" VERTICAL TOGGLE CLAMP 450lbs capacity AC55552 ...$27.95 1-9/16" VERTICAL TOGGLE CLAMP 700lbs capacity AC55555....$28.95 The versatile Wonder Dog clamp can be used anywhere you can drill a 3/4" hole. Using a Wonder Dog in combination with a standard round bench dog, you can make a clamp of any length by drilling 3/4" holes in a 2 x 4, 2 x 6, etc. At the lowest projection, only 5/8" of the dog head is above the bench. The swivel handle lets you apply ample clamping pressure in any position. The dog is drilled and tapped at a 3° angle to ensure that work is held tight to the bench. The Wonder Pup is the same as the Wonder Dog in all respects, except that the post is shorter (2-3/4" instead of 6-1/2") and has no side spring. WONDER DOG VT05101 ..........................$39.95 WONDER PUP VT05102 .........................$36.95 13/16" HORIZONTAL TOGGLE CLAMP 450lbs capacity AC55020 ................$24.95 2" HORIZONTAL TOGGLE CLAMP 700lbs capacity AC55050 ..................$28.95 2-3/4" HORIZONTAL TOGGLE CLAMP 700lbs capacity AC55070 .................$29.95 Veneer Press Clamp Although considered the clamp of choice for making a veneer press, these clamps are also ideal for any number of custom fixtures for all types of gluing, veneering, welding and assembly. Overall length is 11-1/2". GearKlamp 3/8" INLINE TOGGLE CLAMP 450lbs capacity AC55015 ..............$28.95 5/8" IN-LINE TOGGLE CLAMP 700lbs capacity AC55025 ..............$29.95 The new GearKlamp provides a unique clamping solution in tight spaces and everywhere else quick and secure clamping is needed. Most clamps use a tightening handle which is in line with the clamping pads, making them hard to tighten in tight spots. The GearKlamp can be easily moved into position using the quick release button and tightened using the ratcheting handle located on the bar. This is the ideal clamp to use in cramped locations, when up against another object or when reaching into a space to clamp. BESSEY GEAR CLAMP 6" x 2-1/2" AC20442 ..............$22.95 12" x 2-1/2" AC20443 ..............$26.95 18" x 2-1/2" AC20444 ..............$28.95 24" x 2-1/2" AC20445 ..............$29.95 VENEER PRESS CLAMP WK28930 ....................$14.95 23' Strap Clamp Toggle Clamps The Bessey variable angle strap clamp has two distinguishing features that separate it from other generic strap clamps. First, the geared winder simultaneously tightens both sides of the strap. Secondly, the built-in rewinding mechanism keeps the strap rolled up and out of the way. The 4 clamp pads that come with the strap clamp can handle any angle from 60 to 180 degrees. STRAP CLAMP AC05569 ................................................$36.95 200 LB VERTICAL HANDLE TOGGLE CLAMP TGC-001 ....................$9.95 500 LB VERTICAL HANDLE TOGGLE CLAMP TGC-003 ..................$12.95 750 LB VERTICAL HANDLE TOGGLE CLAMP TGC-005 ..................$19.95 Bessey Track Clamp Set • With special guide on the top section for grooves 12 x 6.5 mm to 12 x 8 mm • Use with adapter on the bottom section and rotated top section for secure fastening of guide rails from Festool, Protool, Metabo, Makita, Hitachi/Hikoki, Dewalt, etc. and for clamping in profiles and on work benches • Use with adapter on the top or bottom section for step-over clamping • Use with or without adapter – for clamping or spreading • Clamping force up to 100 lbs • Ergonomically shaped 2-part plastic handle with pump lever behind the rail BESSEY TRACK CLAMP SET AC20500......$36.95 500 LB HORIZONTAL HANDLE TOGGLE CLAMP TGC-007 ..................$12.95 750 LB HORIZONTAL HANDLE TOGGLE CLAMP TGC-008 ..................$18.95 4" Framing Vise This cast-iron Picture Framing Vise has a 4" capacity for securing a wide range of frame sizes. Designed to permanently mount to a benchtop, the Framing Vise features 3-point leveling, tilt, and 360° swivel. Large rubber- gripped handles make clamping quick and easy. Overall size is 3-3/4" x 3-3/4" x 5-3/4" tall and weighs 12.4 lbs. 4" FRAMING VISE WK32270......................$59.95 300 LB PUSH-PULL CLAMP TGC-009 ....$15.95 800 LB PUSH-PULL CLAMP TGC-010 ....$20.95
Table of Contents Order Form Catalog Index Visual Index
Catalog Search:
---Select--- Catalog Home 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
Put your note here.
Move delete
Magnified View - Please use mouse to move page for easier readability
Catalog Select | Catalog Home | Contact Us | Catalog Request | Visual Index | Mobile
PaperCatalogsOnline
www.papercatalogsonline.com
2703 Windswept Lane, Suite 100
Online Catalog Format © 2021 PaperCatalogsOnline
Copyright 2021 - Powered by PaperCatalogsOnline


text:
\section{Introduction}

\begin{figure*}[ht]
\vskip 0.2in
\begin{center}
\centerline{\includegraphics[scale=0.9]{pic/xlp.pdf}}
\caption{\textbf{An illustration of the language embedding approach (additive, left) and our proposed Cross-lingual Language Projection (XLP, right)}. For a sentence from language $t$, the language embedding $l_t$ is added to each word embedding and then fed into the Transformer layers. XLP projects word embeddings into language-specific semantic space using projection function $P_t$. Then the projected embeddings will be fed into the Transformer model to process with their language-specific meanings. In both figures, we omit the positional embedding for a better comparison of the two methods.}
\label{fig:xlp-illustration}
\end{center}
\vskip -0.2in
\end{figure*}

Learning a universal NLP model that supports multiple languages is usually a necessity in practical systems, such as multilingual machine translation, and multilingual sentence understanding ~\cite{dong-etal-2015-multi,zoph2016multisource,firat2016multiway,wu2016googles,johnson2017googles,lee2017fully,firat2016zeroresource,gu2018universal,lewis2020mlqa,lample2019crosslingual,conneau2020unsupervised}. Transformer~\cite{vaswani2017attention} is the most widely used neural network architecture in natural language processing. To handle sentences from different languages, people design ways to provide the ``language'' signal into the multilingual Transformer. Language embedding \cite{tan2019multilingual,lample2019crosslingual,huang2019unicoder,chi2019crosslingual,liu2020multilingual,tang2020multilingual} is a popular choice which views each language as a symbol with a learnable embedding vector. Previous works provide two approaches to using the language symbol: attaching it to the beginning of the sentence or adding its embedding to the word embedding at each position. Using such information, Transformers try to learn the word meanings in the corresponding language and obtain the contextual word representations accordingly.

In this work, we revisit the use of language embedding in Transformer and find that the current approaches may be ineffective in learning multilingual representations. To show the problem clearly, we study the interaction between word embedding and language embedding in the Transformer layers and find that in both approaches, the word-language correlation will be computed in the self-attention module. We question whether this word-language correlation is useful in capturing semantic relations of words at different positions in the sentence. According to our empirical study, we observe that this correlation seems to reflect the popularity of a word appearing in a language to a certain extent. Obviously, such popularity cannot reflect whether two words have a strong semantic relationship in a language.

The analysis inspires us to think further about the proper way to encode ``language'' in a multilingual Transformer model. As the vocabulary is shared (i.e., one word-unit corresponds to one embedding vector), one word-unit may appear in multiple languages and have different meanings. We hope that with the language encoding, the Transformer model can receive language-specific word meaning and learn the contextual word representation based on that. Motivated by this, we propose a novel language encoding called Cross-lingual Language Projection (XLP). Instead of viewing ``languages'' as vectors, we view different ``languages'' as different projection functions, e.g., linear transformation matrice with learnable parameters. Given any sentence, XLP projects the word embeddings into language-specific semantic space using the corresponding projection function. Then the Transformer takes the projected word embeddings as input, calculates word-word correlations in the language-specific semantic space, and obtains the representation of the words and the sentence. See Figure \ref{fig:xlp-illustration} for an illustration.

XLP is conceptually simple and easy to implement with barely additional computation overhead but shows promising performance gain on a wide range of multilingual applications. To be specific, XLP gains 1.2\% of accuracy improvement on the zero-shot cross-lingual task XNLI \cite{conneau2018xnli} comparing to the previous state-of-the-art pre-trained model. XLP also shows better performance on multilingual translation tasks. It consistently improves the BLEU scores on translating different languages to English compared to the baseline models.

\section{Preliminary}

\subsection{Attention Module}

The attention module \citep{vaswani2017attention} is formulated as querying a dictionary with key-value pairs, e.g., $\text{Attention}(Q,K,V)=\text{softmax}(\frac{QK^T}{\sqrt{d}})V$, where $d$ is the dimensionality of the hidden representations, and $Q$ (Query), $K$ (Key), $V$ (Value) are specified as the hidden representations of the previous layer. The multi-head variant of the attention module is popularly used which allows the model to jointly attend to the information from different representation sub-spaces, and is defined as
\begin{align}
\text{Multi-head}(Q,K,V) ={}& \text{Concat} (\text{head}_1,\cdots,\text{head}_H)W^O \nonumber\\
\text{head}_k ={}& \text{Attention}(QW_k^Q, KW_k^K,VW_k^V), \nonumber
\end{align}
where $W_k^Q\in \mathcal{R}^{d \times d_K}$, $W_k^K\in \mathcal{R}^{d \times d_K}$, $W_k^V\in \mathcal{R}^{d\times d_V}$, and $W^O\in \mathcal{R}^{H d_V \times d}$ are learnable project matrices, $H$ is the number of heads. $d_K$ and $d_V$ are the dimensionalities of Key and Value.

The self-attention module is one of the key components in Transformer and BERT encoder \citep{devlin2019bert}. For simplicity, we use the single-head self-attention module and set $d_K = d_V = d$ for a demonstration. We denote $x^l=(x^l_{1}, x^l_{2} \cdots, x^l_{n})$
as the input to the self-attention module in the $l$-th layer, where $n$ is the length of the sequence and each vector $x^l_i \in \mathcal{R}^d$ is the contextual representation of the token at position $i$. $z^l=(z^l_{1}, z^l_{2} \cdots, z^l_{n})$ is the output of the attention module. Then, the self-attention module can be written as
\begin{eqnarray}\label{eqn:general-att2}
z^l_i=\sum_{j=1}^n \frac{\exp (\alpha_{ij})}{\sum_{j'=1}^n\exp (\alpha_{ij'})}(x^l_jW^{V,l}), \\
\text{where } \alpha_{ij}=\frac{1}{\sqrt{d}} (x^l_iW^{Q,l})(x^l_jW^{K,l})^T.
\label{eqn:self-attn}
\end{eqnarray}
As we can see, in any sentence, the self-attention module calculates the correlation between information at different positions, and use the correlation (i.e., attention) to obtain the contextual representation of each word by considering its surroundings.

\subsection{Language Encoding in Multilingual Transformer}
When training a multilingual model, a shared vocabulary (of words or sub-words) covering all the languages is firstly prepared. A learnable word embedding is assigned to each word in the vocabulary. Then a Transformer model, which takes the word embeddings (and the positional embeddings) as input, is optimized using pre-defined objective functions on the multilingual training data. For example, in multilingual machine translation, an encoder-decoder Transformer is trained to maximize the conditional log-likelihood of the target sentence given the source sentence using translation pairs in all the languages.

However, we usually need to feed the model with an additional signal in terms of which language the sentence comes from. Sometimes, such information is essential. For example, in multilingual machine translation, the model can generate the translation results only if we provide the name of the target language we are requesting. To encode such information, previous works design a specific symbol for each language with a learnable embedding vector. There are generally two approaches to use language embeddings. The first approach (attaching approach for short) attaches the corresponding symbol to the beginning of the sentence \citep{wu2016googles,johnson2017googles,liu2020multilingual,tang2020multilingual}. The second approach (additive approach for short) adds the language embedding to the word embedding at each position \citep{tan2019multilingual,lample2019crosslingual,huang2019unicoder,chi2019crosslingual}. With the help of language embeddings, the model receives the ``language'' information explicitly from the input and learns the sentence representations through Transformer layers.

\section{Cross-Lingual Language Projection}

\begin{figure}[ht]
\vskip 0.2in
\begin{center}
\centerline{\includegraphics[width=0.5\textwidth]{pic/sample_correlation.pdf}}
\caption{\textbf{Visualizations of the word-language correlation in Eq.\eqref{eqn:expand-attn}.} We use the publicly released pre-trained XLM model and calculate the correlations on one head. We sampled two synonymous sentences (English (left) and Chinese (right)) from the Wikipedia data corpus in the figure for illustration. In each bar chart, the $i$-th element is the correlation calculated between the $i$-th word and the corresponding language embedding. Even though Chinese words are quite different from western languages, they both display the popularity of words to some degree.}
\label{fig:vis-correlation}
\end{center}
\end{figure}

\subsection{Revisiting Language Embedding}
We are interested in the role of the language embeddings in learning multilingual representations through Transformer layers. Assume that there are $N$ languages in the multilingual data corpus. Following the notations in Section 2.1, we denote $l_t \in \mathcal{R}^d$ as the language embedding for the $t$-th language, where $t\in[N]$. Denote $s=(w_1,....,w_n)$ as a sentence in the $t$-th language, where each $w_i\in \mathcal{R}^d$ is the word embedding. It is easy to show that when either applying the attaching approach or the additive approach, the self-attention (Eq.\ref{eqn:self-attn}) will calculate the correlation between word and language embeddings through dot-product. We show this explicitly for the additive approach while the analysis and results for the attaching approach are similar.

In the additive approach, the input\footnote{Usually, positional embedding is another term that will be added to the word embedding in the input. We omit the positional embedding for a better illustration of how language embeddings interact with words. The conclusions will not change if we take the positional embedding together into consideration. Recent works also show that positional embedding is not an essential
term in the input, see \cite{ke2020rethinking,shaw2018self}.}  to the Transformer model is $\{w_i+l_t\}_{i=1}^{n}$. Then, in the self-attention module of the first Transformer layer, the correlation term $\alpha_{ij}$ in Eq.\ref{eqn:self-attn} can be expanded as:
\begin{align}\label{eqn:expand-attn}
    \alpha_{ij} &= \frac{((w_i+l_t)W^{Q,1}) ((w_j+l_t)W^{K,1})^T}{\sqrt{d}} \\
                      &= \frac{(w_iW^{Q,1}) (w_jW^{K,1})^T}{\sqrt{d}} + \frac{(l_tW^{Q,1}) (w_jW^{K,1})^T}{\sqrt{d}} \notag\\
                      &+ \frac{(w_iW^{Q,1}) (l_tW^{K,1})^T}{\sqrt{d}} + \frac{(l_tW^{Q,1}) (l_tW^{K,1})^T}{\sqrt{d}}. \notag
\end{align}
It can be seen that there are four terms in the expansion: word $\rightarrow$ word, language $\rightarrow$ word, word $\rightarrow$ language and language $\rightarrow$ language correlations. The first term characterizes the relationships between a pair of words, and the language embedding involves in the calculation of the other three terms. It is obvious to see that the last term is redundant since the value is a constant for every $i,j$. Adding the same constant to each dimension of the input will not change the output of the Softmax function.

The two terms in the middle calculate the correlations between word embedding and language embedding. In particular, it can be seen from Eq.~\ref{eqn:expand-attn}, any word $w_i$ will calculate with the same language embedding $l_t$ in sentence $s$. We argue such correlations cannot reflect how two words in language $t$ correlate (e.g., have similar meanings) with each other. To study the function of the two terms, we download the officially released XLM15 model \cite{lample2019crosslingual}, and calculate those values on sampled sentences from the Wikipedia data corpus. We showcase the results in Figure \ref{fig:vis-correlation}. Empirically, we observe that the values seem to reflect the popularity of a word appearing in a language to a certain extent, e.g., ``the'' and ``of'' have relatively large dot-products with the language embedding of ``English''. But obviously, word-frequency correlation might not reflect the word-semantic correlation well.

\subsection{From Language Embedding to Language Projection}

The discussion above reveals some issues in the previous approaches and further motivates us to think about a better way to encode language information into a multilingual Transformer model. Note that the model uses a shared vocabulary, and a word-unit (or sub-word) may appear in multiple languages. We argue that for any language, the language encoding should provide the Transformer with language-specific meanings of the words in a sentence.

Our idea is to use language projection instead of language embedding, which can provide language-specific word representations by \textit{projecting} the word embedding to language-specific semantic space. We denote $\phi_t: \mathcal{R}^{d} \rightarrow \mathcal{R}^{d}, t\in[N]$ as the projection function for the $t$-th language. For any sentence $s=(w_1,....,w_n)$ from language $t$ , we first project each word $w_i$ to $\phi_t(w_i)$ which characterizes the semantic meaning of $w_i$ in language $t$. After the projection, the language-specific word embedding $\phi_t(w_i)$ will be used as input to the Transformer model instead of $w_i$. Mathematically, we have the following form in the self-attention calculation:
\begin{equation}
    \label{eqn:language_function}
    \alpha_{ij} = \frac{(\phi_t(w_i)W^{Q,1})(\phi_t(w_j)W^{K,1})^T}{\sqrt{d}}
\end{equation}

We call our method Cross-lingual Language Projection (XLP for short). Linear projection is a popularly used semantic projection in many previous works \cite{vaswani2017attention, conneau2017word} and we use linear projection function in XLP. We define $\phi_t(w_i)=w_i P_t$, where $P_t$ is a $d\times d$ matrix. Then Eq.\ref{eqn:language_function} becomes

\begin{equation}
    \label{eqn:language_projection}
    \alpha_{ij} = \frac{((w_iP_t)W^{Q,1})((w_jP_t)W^{K,1})^T}{\sqrt{d}}
\end{equation}

It is worth noting that introducing $P_t$ can be viewed as a decoupling of the original projection matrices in the self-attention module. Essentially, $P_t$ learns language-specific projection to transform the word embedding to the language-specific semantic space. At the same time, $W^Q$ and $W^K$ still learn to project the semantic information to proper subspace as in the standard monolingual Transformer.

In Figure \ref{fig:xlp-illustration}, we illustrate XLP and compare it with the language embedding (the additive approach). It can be seen that using the language embedding is equivalent to shifting the word embedding space by a language-specific bias. In contrast, our proposed language projection projects the word embedding to language-specific semantic space. Thus, the self-attention module can obtain the language-specific word correlations and learn more efficiently.

\subsection{Implementation and Discussions}

XLP is very easy to implement in any open-source deep learning framework like PyTorch \citep{paszke2019pytorch}. The overall pseudo code is shown in Algorithm \ref{alg:code}. More details are presented as follows.

\begin{algorithm}[t]
\caption{Pseudocode of XLP in a PyTorch-like style.}
\label{alg:code}

\definecolor{codeblue}{rgb}{0.25,0.5,0.5}
\lstset{
  backgroundcolor=\color{white},
  basicstyle=\fontsize{7.2pt}{7.2pt}\ttfamily\selectfont,
  columns=fullflexible,
  breaklines=true,
  captionpos=b,
  commentstyle=\fontsize{7.2pt}{7.2pt}\color{codeblue},
  keywordstyle=\fontsize{7.2pt}{7.2pt},
}
\lstinputlisting[language=python]{pseudo-code.py}
\end{algorithm}

\textbf{Incorporate XLP with the positional embedding.} Positional encoding is an essential component in Transformer since other main components of the model are entirely invariant to sequence order. The absolute positional encoding is the most popularly used one, which provides each position an embedding vector. The positional embedding is added to the word embedding, which is found significantly helpful at learning the contextual representations of words at different positions. In XLP, the language projection is only applied to the word embedding. That being said, for any sentence, we first project the word embedding using XLP and then add it to the positional embedding.

\textbf{The increase of parameters and efficiency.} The language embedding approaches use $N$ $d$-dimentional vectors where $N$ is the number of languages and $d$ is the embedding dimension. XLP uses $N$ $d\times d$ matrice, which is slightly larger than previous approaches. Taking XLM15 \citep{lample2019crosslingual} architecture as an example, the newly introduced language projection parameters are about 15M, which is only about 6\% of the 250M parameters in XLM15. Since XLP doesn't relate to the number of Transformer layers, the increase of parameters can be ignored when using deeper models. In terms of efficiency, XLP only needs an additional linear transformation in the input layer, which introduces barely computational overhead comparing to the stacked Transformer layers.

\section{Experiment}
We conduct extensive experiments on multilingual language understanding and multilingual machine translation tasks to verify our proposed XLP. The codes are implemented based on \texttt{fairseq}\footnote{\url{https://github.com/pytorch/fairseq/}} \citep{ott2019fairseq} and \texttt{XLM}\footnote{\url{https://github.com/facebookresearch/XLM/}} in \texttt{PyTorch} \citep{paszke2019pytorch}. Models are trained on 16/8 NVIDIA Tesla V100 GPUs with mixed-precision \citep{micikevicius2018mixed} for the multilingual language understanding / machine translation tasks respectively.

\subsection{Multilingual Language Understanding}

\begin{table*}[t]
\caption{\textbf{Results on XNLI cross-lingual natural language inference.} We report the accuracy on each language and the averaged accuracy. The languages are sorted by the resource magnitude order. \#lg is the number of languages used in the pre-training corpus.}
\label{tab:XNLI}
\vskip 0.1in
\begin{center}
\begin{small}
\renewcommand\tabcolsep{2.4pt}
\begin{tabular}{l|c|ccccccccccccccc|c}
\toprule
  & \textbf{\#lg} & \textbf{en} & \textbf{fr} & \textbf{es} & \textbf{de} & \textbf{el} & \textbf{bg} & \textbf{ru} & \textbf{tr} & \textbf{ar} & \textbf{vi} & \textbf{th} & \textbf{zh} & \textbf{hi} & \textbf{sw} & \textbf{ur} & \textbf{Avg} \\
\midrule
\multicolumn{17}{l}{\textit{Cross-Lingual Transfer (Fine-tune the multilingual model on English training set)}} \\
\midrule
\citet{conneau2017word} & - & 73.7 & 67.7 & 68.7 & 67.7 & 68.9 & 67.9 & 65.4 & 64.2 & 64.8 & 66.4 & 64.1 & 65.8 & 64.1 & 55.7 & 58.4 & 65.6 \\
\citet{artetxe2019massively} & - & 73.9 & 71.9 & 72.9 & 72.6 & 73.1 & 74.2 & 71.5 & 69.7 & 71.4 & 72.0 & 69.2 & 71.4 & 65.5 & 62.2 & 61.0 & 70.2 \\
\midrule
mBERT \citep{devlin2019bert}  & 102 & 82.1 & 73.8 & 74.3 & 71.1 & 66.4 & 68.9 & 69.0 & 61.6 & 64.9 & 69.5 & 55.8 & 69.3 & 60.0 & 50.4 & 58.0 & 66.3 \\
XLM100 \citep{lample2019crosslingual} & 100 & \textbf{83.7} & 76.2 & 76.6 & 73.7 & 72.4 & 73.0 & 72.1 & 68.1 & 68.4 & \textbf{72.0} & 68.2 & 71.5 & 64.5 & 58.0 & 62.4 & 71.3 \\
XLM15 \citep{lample2019crosslingual} & 15 & 83.2 & 76.5 & 76.3 & 74.2 & 73.1 & 74.0 & 73.1 & 67.8 & 68.5 & 71.2 & 69.2 & 71.9 & \textbf{65.7} & 64.6 & 63.4 & 71.5 \\
\textbf{XLP} & 15 & 83.2 & \textbf{76.6} & \textbf{78.3} & \textbf{74.5} & \textbf{75.5} & \textbf{74.7} & \textbf{73.4} & \textbf{70.0} & \textbf{72.3} & \textbf{72.0} & \textbf{69.3} & \textbf{73.8} & 65.2 & \textbf{67.1} & \textbf{64.9} & \textbf{72.7} \\
\midrule
\multicolumn{17}{l}{\textit{Translate-Train (Fine-tune the multilingual model on each training set)}} \\
\midrule
XLM100 \citep{lample2019crosslingual} & 100 & 82.9 & 77.6 & 77.9 & 77.9 & 77.1 & 75.7 & 75.5 & 72.6 & 71.2 & 75.8 & 73.1 & 76.2 & 70.4 & 66.5 & 62.4 & 74.2\\
\textbf{XLP} & 15 & \textbf{83.2} & \textbf{79.3} & \textbf{79.9} & \textbf{78.4} & \textbf{78.7} & \textbf{78.6} & \textbf{76.4} & \textbf{75.0} & \textbf{75.6} & \textbf{76.2} & \textbf{74.7} & \textbf{77.7} & \textbf{70.7} & \textbf{70.8} & \textbf{64.4} & \textbf{76.0}\\
\bottomrule
\end{tabular}
\end{small}
\end{center}
\vskip -0.1in
\end{table*}

\textbf{Model configurations.} For a fair comparison, we implement XLP based on XLM15 \citep{lample2019crosslingual} (250M parameters) architecture. Specifically, XLP consists of 12 Transformer encoder layers. For each layer, the dimensions of hidden representation and feed-forward layer are set to 1024 and 4096, respectively. The number of heads in the attention module is set to 8. XLM15 model supports 15 languages, and we use 15 projection matrices with shape 1024$\times$1024 in XLP.

\textbf{Pre-training.} Following \citet{lample2019crosslingual}, we use Wikipedia corpus (15 languages) for pre-training. The detailed descriptions of the dataset can be found in Appendix A. We perform a couple of consecutive pre-processing steps following \citet{lample2019crosslingual}: normalizing, lower-casing, tokenizing the texts by Moses decoder\footnote{\url{https://github.com/moses-smt/mosesdecoder}} \citep{koehn-etal-2007-moses} and finally applying the byte pair encoding (BPE)\footnote{\url{https://github.com/glample/fastBPE}} \citep{sennrich2016neural} with the same codes (size 80k) and vocabulary (size 95k) from the XLM15. We use masked language modeling \citep{devlin2019bert} as the training objective. The model is trained for 750k steps. The batch size is set to 64 per GPU,  which is the same as the XLM \citep{lample2019crosslingual}. The gradient is accumulated every 4 optimization steps. Detailed description of the settings are presented in Appendix B.1.

\textbf{Fine-tuning.}  We use XNLI (\textbf{Cross}-lingual \textbf{N}atural \textbf{L}anguage \textbf{I}nference) \citep{conneau2018xnli} as the downstream evaluation benchmark to compare our proposed model with the baselines. Given a premise sentence and a hypothesis sentence in a language, the goal of the task is to predict whether the premise entails the hypothesis (entailment), contradicts the hypothesis (contradiction), or neither (neutral). XNLI dataset contains 15 languages, including low-resource languages such as Swahili and Urdu.

Following \citet{lample2019crosslingual}, we evaluate the pre-trained models on the XNLI tasks in two settings. The first setting is called Cross-Lingual Transfer, in which we fine-tune the model on the English training set and evaluate it on the test sets of all languages. The second setting is called Translate-Train, in which we fine-tune and evaluate the model on the dataset of each language, respectively. For all the downstream experiments, we strictly follow \citet{lample2019crosslingual} for the hyperparameter configuration and search space using the official script\footnote{\url{https://github.com/facebookresearch/XLM/\#fine-tune-your-xlm-model-on-cross-lingual-classification-xnli}}.

\textbf{Results.}  The fine-tuning performance on XNLI is presented in Table \ref{tab:XNLI}. Languages are ordered according to the resource magnitude. We compare our model with five baselines: an LSTM-based model~\citep{conneau2018xnli}, a supervised model trained using translation pairs~\citep{artetxe2019massively}, the multilingual BERT model (mBERT)~\citep{devlin2019bert} and officially released XLM models (XLM15, XLM100). For all baselines, we use the number reported in their original papers.

It can be easily seen that XLP outperforms all baselines significantly. In the Cross-Lingual Transfer setting, XLP obtains 72.7\% averaged accuracy, which outperforms the XLM \citep{lample2019crosslingual} and mBERT \citep{devlin2019bert} by 1.2\% and 6.4\% respectively. When we fine-tune the pre-trained model on each language respectively (Translate-Train), XLP still achieves 1.8\% improvement over XLM.

Moreover, XLP obtains a more balanced performance over the 15 languages. For high-resource languages, our approach is competitive (in English) or slightly better (in French, Greek)  than previous works. For extremely low-resource languages such as Swahili and Urdu, XLP outperforms XLM by a significant margin (2.5\% and 1.5\%). All the results indicate that our proposed XLP can help the model learn the multilingual sentence representation better.

\subsection{Multilingual Machine Translation}

\begin{table*}[t]
\caption{\textbf{BLEU scores of 6 languages $\rightarrow$ English on IWSLT dataset.} Language embedding approaches are compared as baselines. The attaching approach attaches the language embedding to the beginning of the sentence, while the additive approach adds the language embedding to the word embedding at each position.}
\label{tab:multi-trans}
\vskip 0.1in
\begin{center}
\begin{small}
\begin{tabular}{l|rrrrrr|r}
\toprule
\textbf{Model} & \textbf{ar-en} & \textbf{de-en} & \textbf{es-en} & \textbf{fr-en} & \textbf{tr-en} & \textbf{zh-en} & \textbf{Avg}\\
\cmidrule(lr){1-1} \cmidrule(lr){2-2} \cmidrule(lr){3-3} \cmidrule(lr){4-4} \cmidrule(lr){5-5} \cmidrule(lr){6-6} \cmidrule(lr){7-7} \cmidrule(lr){8-8}
Language Embedding (attaching) & 30.5 & 33.7 & 40.1 & 41.9 & 23.3 & 18.5 & 31.3 \\
Language Embedding (additive)  & 30.6 & 33.8 & 40.1 & 42.0 & 23.2 & 18.4 & 31.4 \\
\textbf{XLP}  & \textbf{31.0} & \textbf{34.4} & \textbf{40.6} & \textbf{42.2} & \textbf{24.2} & \textbf{18.7} & \textbf{31.9} \\
\bottomrule
\end{tabular}
\end{small}
\end{center}
\vskip -0.1in
\end{table*}

\begin{figure}[ht]
\centering
\vskip 0.1in
\begin{subfigure}{.23\textwidth}
  \centering
  \includegraphics[width=1.46in]{pic/pretrain-curve.eps} \vskip -0.01in
  \caption{Valid loss (Pre-training).}
  \label{fig:pretrain-loss}
\end{subfigure}
\begin{subfigure}{.23\textwidth}
  \centering
  \includegraphics[width=1.50in]{pic/translate-curve.eps}
  \caption{Valid loss (Translation).}
  \label{fig:translate-loss}
\end{subfigure}
\caption{\textbf{Validation loss curve of different methods.}
We plot the validation loss curve in both the cross-lingual language model pre-training and the multilingual translation task. It can be seen that XLP trains faster than previous approaches in both tasks.}
\label{fig:curve}
\vskip -0.1in
\end{figure}

\textbf{Model Configurations.} Following \citet{vaswani2017attention}, we use a 6-layer encoder-decoder-based Transformer in the machine translation tasks. The dimension of the hidden representation and the feed-forward layer is set to 512 and 1024 respectively. The number of the heads in the attention module is set to 4. We evaluate two language embedding approaches described in Section 2.2 as our baseline,  the attaching approach and the additive approach. All experiments use the same training and inference configurations. Detailed description of the settings are presented in Appendix B.2.

\textbf{Datasets.} We collect 6 languages $\leftrightarrow$ English translation pairs from IWSLT evaluation campaign\footnote{\url{https://wit3.fbk.eu/2014-01}} (IWSLT 2014). The details about the datasets can be found in Appendix A. All the sentences are first tokenized with Moses tokenizer and then segmented into subword symbols using BPE. We learn the BPE merge operations across all the languages by setting the size of the BPE codes to 30000 and obtain a joint vocabulary with size 38413.

\textbf{Training and Inference.} We concatenate all the datasets to train a universal multilingual translation model. The mini-batch size is set to 4096 tokens per GPU. We use Adam \citep{kingma2017adam} as the optimizer, and set the hyperparameter $\epsilon$ to 1e-8 and ($\beta1,\beta2$) to (0.9, 0.98). The peak learning rate is set to 5e-4 with a 4k-step warm-up stage followed by an inverse square-root learning rate scheduler. We set the dropout probability to 0.3 and weight decay to 1e-4. Label smoothed cross-entropy is used as the objective function by setting $\epsilon = 0.1$ \citep{szegedy2015rethinking}. The number of training epoch is set to 180. We evaluate the translation quality by tokenized BLEU with sacreBLEU \footnote{\url{https://github.com/mjpost/sacrebleu}} \citep{post-2018-call}.

\textbf{Results.} The multilingual translation results are presented in Table \ref{tab:multi-trans}. First, our proposed XLP consistently outperforms the language embedding approaches (attaching or additive) and achieves an average 0.5 BLEU improvement comparing to the language embedding approaches. Besides, the experimental results also show that XLP works better on the low-resource \textit{TR} $\rightarrow$ \textit{EN} dataset (1.0 BLEU improvement). The overall comparison of the multilingual translation task also demonstrates the effectiveness of our proposed XLP.

\subsection{Discussion}

\textbf{Training Efficiency.}  By using the language projection, the Transformer model can receive concrete language-specific semantic information as input, which makes the model easier to train. To show this, we study the validation loss curves of XLP and the baselines.

In cross-lingual language model pre-training, since XLM \citep{lample2019crosslingual} did not release any intermediate model checkpoints, we pre-trained both the XLM and the XLP using the same pre-training hyperparameters and check the intermediate model performance. In Figure \ref{fig:pretrain-loss}, we plot the pre-training validation loss of XLM and XLP for the first 350K steps. In multilingual machine translation, we compare the validation loss curve of XLP and previous language embedding approaches. The result is shown in Figure \ref{fig:translate-loss}. It can be easily seen that our proposed XLP
can use fewer steps to achieve comparable validation loss than previous approaches.

\begin{figure*}[ht]
\begin{subfigure}{.5\textwidth}
  \centering
  \includegraphics[width=2.7in]{pic/xlm-batch-vis.pdf}
  \caption{XLM}
  \label{fig:xlm-batch-vis}
\end{subfigure}
\begin{subfigure}{.5\textwidth}
  \centering
  \includegraphics[width=2.7in]{pic/xlp-batch-vis.pdf}
  \caption{XLP}
  \label{fig:xlp-batch-vis}
\end{subfigure}
\caption{\textbf{Cosine similarity of processed words across different languages.} The word embeddings are processed using language embeddings in XLM (a) and XLP (b) }
\label{fig:vis-xlm-xlp}
\end{figure*}

\begin{figure*}
\begin{subfigure}{.5\textwidth}
  \centering
  \includegraphics[width=2.7in]{pic/en-xlm-vis.eps}
  \caption{XLM}
  \label{fig:en-xlm-vis}
\end{subfigure}
\begin{subfigure}{.5\textwidth}
  \centering
  \includegraphics[width=2.7in]{pic/en-xlp-vis.eps}
  \caption{XLP}
  \label{fig:en-xlp-vis}
\end{subfigure}
\caption{\textbf{Cosine similarity of different processed words in ``English''.} The word embeddings are processed using language embeddings in XLM (a) and XLP (b).}
\label{fig:vis-xlm-xlpcorss-words}
\end{figure*}

\textbf{Cross-Lingual Transfer Gap.} The XTREME benchmark \citep{hu2020xtreme}  proposed to use the cross-lingual transfer gap to evaluate the multilingual models. The gap is measured by the difference between the performance on the English data and the averaged performance on all other language data. In such a way, we can estimate how powerful the multilingual model is when transferring its knowledge from English to other languages. As shown in Table \ref{table: xTansGap}, in terms of this metric, our proposed XLP significantly outperforms the mBERT, XLM100 and XLM15 by 5.7\%, 2.7\% and 1.3\% respectively on XNLI.

\begin{table}[t]
\caption{\textbf{Cross-lingual Transfer Gap Score.} The score is calculated as the difference between the performance on the English data and the averaged performance on all other language data in XNLI.}
\vskip 0.2in
\label{table: xTansGap}
\begin{center}
\begin{small}
\begin{tabular}{lc}
\toprule
\textbf{Models} & \textbf{XNLI} \\
\midrule
mBERT \citep{devlin2019bert}  & 16.9 \\
XLM100 \citep{lample2019crosslingual}   & 13.9 \\
XLM15 \citep{lample2019crosslingual}   & 12.5 \\
XLP    & \textbf{11.2} \\
\bottomrule
\end{tabular}
\end{small}
\end{center}
\vskip -0.2in
\end{table}

\subsection{Visualization.}
To better understand the learned language projections in XLP, we design methods to visualize them. Through the experiments,  we empirically verify that the language-specific projections indeed encode the language information and help the model efficiently capture the language-specific word correlations, while language embeddings do not.

First, we investigate whether the language encoding approaches learn language-specific information. The high-level idea is that if a language encoding approach learns languages specific information, the embeddings of a word processed by the language encoding approach should be different for different languages. To show this, we compare the released XLM15 model with our pre-trained XLP model for 15 languages. We first sample some English words (sub-words) from the multilingual vocabulary. For each word, we process the word embedding by different language encoding approaches (additive v.s. projection) and obtain language-specific embeddings of the same word. We then calculate the cosine similarity of these embeddings to see whether they are similar, which forms a $15\times15$ similarity matrix for each model.

As shown in Figure \ref{fig:xlm-batch-vis}, the elements in the similarity matrix of XLM are surprisingly high, while for XLP in Figure \ref{fig:xlp-batch-vis}, the element values are quite diverse. This demonstrates that our language projection indeed projects the word embeddings to different semantic spaces as we expect.

Taking one step further, we investigate whether the language encoding can help the model capture the word correlations. We select three topics first and then select three English words in each topic: [\textit{happy}, \textit{glad}, \textit{sad}], [\textit{car}, \textit{plane}, \textit{bike}] and [\textit{meat}, \textit{food}, \textit{rice}]. We process the word embeddings into ``English'' with different language encoding approaches (additive v.s. projection), and then compute the cosine similarity of the processed words. The word similarities are presented in Figure \ref{fig:en-xlm-vis} and \ref{fig:en-xlp-vis}. From the two figures, we can see that for XLP, the words in the same topic have strong correlations, while for words in different topics, the correlations are weak. However, in XLM, word pairs seem to have similar correlations. This suggests our proposed XLP captures the correlation of words better.

\section{Related Work}
Before the development of the Transformer model, Google built the first multilingual neural machine translation system (GMNMT) \citep{johnson2017googles} based on LSTM networks \citep{hochreiter1997long}, and introduced the language symbol with the attaching approach. Soon, this method is adopted to the Transformer-based multilingual translation system \citep{liu2020multilingual,tang2020multilingual}. XLM \citep{lample2019crosslingual} proposed to add the language embedding to word embedding at each position for multilingual language understanding tasks. \citet{huang2019unicoder,chi2019crosslingual} follow this language encoding approach further and develop more self-supervised objective functions.

There are several works studying the language-specific and language-agnostic parameters in a universal multilingual model. To build a multilingual machine translation system, early works aimed to increase the shared model capacity from the separated bilingual models to enhance the cross-lingual transfer \citep{dong-etal-2015-multi,zoph2016multisource,lee2017fully,firat2016multiway}. After the success of the first universal multilingual machine translation system \citep{wu2016googles,johnson2017googles}, emergent works \citep{blackwood2018multilingual,V_zquez_2019,escolano2020multilingual} started to investigate which components should be shared between languages and which components should be kept as language-specific in a universal multilingual model.

Recently, an interesting concurrent work \citep{zhang2021share} provided a comprehensive empirical study on the language-specific capacity in a universal multilingual machine translation model. This work suggests that using mixed language-specific and language-agnostic parameters in every sub-layers of the Transformer model is a better choice, letting the model learn to control the shared capacity by itself. However, this work does not take the language encodings into consideration but focuses on the upper Transformer layers. Being complementary to \citet{zhang2021share}, we investigate the previous language embedding approaches and propose to use the language-specific projections for better language encoding.

\section{Conclusion}
In this paper, we revisit the use of language embedding in the multilingual Transformer and identify several problems in the existing formulations. We propose a new approach called Cross-lingual Language Projection (XLP)  to address the issues. XLP uses language-specific transformations to project the word embeddings into language-specific semantic space, which achieves the purpose of appropriately encoding ``language'' in a multilingual Transformer. Extensive experiments demonstrate that the multilingual Transformer models using our proposed XLP consistently outperform those with previous language embedding approaches on multilingual language understanding and machine translation benchmarks.

redpajama_set_name: RedPajamaArXiv


text:
Sidewinders - Flatfoot Hustlin'
Label: Great Eastern Production Co. SR-10
Origin: Halifax, Nova Scotia
Genre: funk
Value of Original Title: $1,500.00
Make Inquiry/purchase: email ryder@robertwilliston.com
Release Type: Albums
Playlist: Canadian as Funk, 1970's, $1000 Record Club, Nova Scotia, Top 50 Collector Albums, Rarest Canadian Music, Holy Grails, Honorary Canadians, MOCM Top 1000 Canadian Albums, Top 50 Black Canadian Albums
1 I Like to Dance 04:52
2 Time For Loving 04:24
3 A Voyage 01:06
4 I Like Your Stuff 04:49
1 Flat Foot Hustlin' 06:16
2 Gift to the Sun 06:09
3 Can't You Feel the Fire 04:55
Sidewinders-Flatfoot Hustlin BACK
Sidewinders-Flatfoot Hustlin LABEL 01
Flatfoot Hustlin'
Information/Write-up
The Sidewinders were a seven piece Halifax, Nova Scotia ensemble who released this self-produced heavy funk album; one of the best Canadian funk albums ever recorded.
Raymond Myers: saxophone
Jerry Holland: keyboards
Jerry Jenkins: guitar
Hank Anderson: bass
Alexander Link and Jerry Ayers: trumpet
Reggie Barnes: drum.
Recorded at Audio Atlantic in Halifax, Nova Scotia.
Engineered by Grant Kennedy
Cover design by Donald C. Gough
All selections were composed by Reggie Barnes, Jerry Ayers, Hank Anderson, Jerry Holland, Jerry Jenkins, Raymond Myers, Alexander Link; except "Voyage", which was written by Al Feeney.
All selections produced by Sidewinders except "I Like to Dance", "I Like Your Stuff", and "Can't You Feel the Fire", which were produced by Andrew Terry.
"For more than ten years now the Sidewinders have been gaining a name as one of the most creative and most exciting groups.
Comprising seven capable and qualified musicians, these young men have backed the biggest of stars and played places reserved for the top names in show business. Realizing a need for greater recognition and exposure, these determined young men set out to record and produce an album. Hard times, trials and tribulations have stretched the making of this album into a three-year period.
But putting both feet firmly on the ground and hustling hard, the album is finally ready for the world to hear. Thus the name "Flat Foot Hustlin'".
"It has been said, that time tells all things, and I'm sure time will include the Sidewinders as a major musical force."
-Natalie MacFarlane
Re-issued by Phantom Records on CD in Japan in 2007.
Our never-ending quest for the ultimate Holy Grail LP has encountered many precious rarities and sought-after gems on the way, and The Sidewinders' Flatfoot Hustlin' is no exception. Incredibly rare, so much so that it's only become known to collectors in recent years, here is another fantastic example of top notch funk n' soul from the '70s that would surely be in everybody's record collection - if only it had been pressed in numbers and properly released! It was a bitterly cold night in Nova Scotia, Canada (of all places?!) when the Sidewinders got together in an unheated studio for a couple of days to cut Flatfoot Hustlin' their first and only album. A professional touring band that had been on the circuit for many years, their sound was slick, solid and smooth. The recording was a wrap, a release was imminent - but then things began to go wrong... Our detailed liner notes from our own exclusive interviews reveal the story behind The Sidewinders and how this album came to be - or not as the case may be! What certainly is the case is that 'Flatfoot Hustlin'' is a great-sounding rare funk album chock full of deep soulful moments and full-on fat, funky grooves.


text:
Abderrahime Bouramdane (born 23 June 1978) is a Moroccan long-distance runner who competes in marathons. He represented Morocco at the 2008 Beijing Olympics and 2011 World Championships in Daegu, South Korea. After winning his first two marathons in Tunis and Marrakesh, Bouramdane represented his country at the World Championships in Athletics in 2005 and 2007.

He won his first international medal at the 2005 Jeux de la Francophonie, taking the marathon bronze. He won the 2006 National Capital Marathon and had a series of second-place finishes, which included runner-up spots at the JoongAng Seoul Marathon and 2008 Boston Marathon. He placed in the top five at the New York City Marathon in 2008 and 2009. His personal best of 2:07:33 hours was set at the 2010 London Marathon. This time ranks him as the fifth fastest Moroccan over the marathon distance.

Career
He grew up in Fes and, inspired by the exploits of Olympic marathon medalist Rhadi Ben Abdesselam, he took up running in 1993. He won his first two top level marathon races, breaking the course record to win the 2004 Tunis Marathon in a time of 2:15:38 hours, and then improving to 2:15:16 to win the Marrakesh Marathon. He gained selection for the 2005 World Championships in Athletics in Helsinki, but he did not manage to finish the race, dropping out after the 35 km. He formed part of a Moroccan medal sweep of the marathon 2005 Jeux de la Francophonie, taking the bronze medal behind Zaïd Laaroussi and Rachid Kisri.

He ran at Ottawa's National Capital Marathon in May 2006 and set a personal best of 2:12:18 hours. He was not the first to cross the line, but a large group of runners had inadvertently taken a shorter route mid-race and, as the first runner to complete the full course, Bouramdane was declared the winner while nearly a dozen athletes were disqualified. His second run of the year was also in Canada: at the Toronto Waterfront Marathon he and Daniel Rono quickly became the two race protagonists. Strong winds put an end to their challenge to the Canadian all-comers record, but Bouramdane improved his best to 2:10:41 hours as the clear runner-up by a margin of over three minutes.

He knocked a second off this time at the 2007 Ottawa race but was again second placed behind a Kenyan runner, this time David Cheruiyot. Another national selection saw him take 45th place at the 2007 World Championships in Athletics, one place behind team mate Kisri, but the premature exits of three other Moroccans meant the country did not rank in the World Marathon Cup competition. The JoongAng Seoul Marathon in November saw Bouramdane duel against Samuel Chelanga for the title and although his rival won the competition, the Moroccan dipped under two hours and ten minutes for the first time for a best of 2:08:20 hours.

The 2008 Boston Marathon saw him take his fourth consecutive runner-up spot on the road circuit behind a Kenyan elite runner, as he recorded his second sub-2:10 time behind Robert Cheruiyot. His Olympic debut followed in August at the 2008 Beijing Olympics, coming 26th overall. His final big race of the year came at the New York City Marathon, where he was fifth.

He failed to finish the Lake Biwa Marathon in February 2009, and turned shorter races instead, placing in the top ten at the Nordion 10km in Ottawa. Having missed out on selection for the 2009 World Championships, he returned to the New York Marathon later that year and took fifth place for a second year running. A personal best came at the 2010 London Marathon, where he finished in 2:07:33 hours to take fourth place. He went on to race in the BUPA Great North Run in September 2010 and finished with another personal best, this time in the half marathon, of 1:02:40. He was somewhat slower at the 2010 New York City Marathon, however, but finished within the top ten.

He opened 2011 with a run at the Lisbon Half Marathon, finishing 1:03:47 for the distance and ninth overall. He had the third fastest run of his career at the 2011 London Marathon as he took seventh position in a time of 2:08:42 hours, finishing just behind fellow Moroccan Jaouad Gharib.  He qualified for the 2012 Summer Olympics, but did not finish the race.

He was banned from competition for two years for irregularities in his biological passport with the ban ending on 1 December 2017. In addition, all his results from 14 April 2011 until the end of the sanction were annulled.

Statistics

International competition

Personal bests
Half marathon – 1:03:47 hrs (2011) 1:02:40 hrs downhill (2010)
Marathon – 2:07:33 hrs (2010)

References

External links

 Abderrahime Bouramdane at Marathon Info

Living people
1978 births
People from Fez, Morocco
Moroccan male long-distance runners
Moroccan male marathon runners
Athletes (track and field) at the 2008 Summer Olympics
Athletes (track and field) at the 2012 Summer Olympics
Olympic athletes of Morocco
Doping cases in athletics
Moroccan sportspeople in doping cases
20th-century Moroccan people
21st-century Moroccan people

redpajama_set_name: RedPajamaWikipedia


text:
How to pick the next cool tech stock
Ryan Lightfoot-Aminoff 04/02/16 in Specialist investing
Who doesn't wish they had the nous to suss out Google as a star performer back when Larry Page and Sergey Brin were inventing its first incarnation—bizarrely called BackRub―as young 20-somethings at Stanford.
But investing in technology no longer means finding the next Facebook while its founder is still working in a garage and wearing tracksuit bottoms. Instead, trend hunters need to check out businesses in the healthcare, energy and manufacturing space, in addition to the more 'traditional' avenues.
Think about all the tech developments you take for granted these days, which used to be A-MAZ-ING.
For instance, say you're exploring a new city and you get lost. Ten years' ago, if you didn't have a map on you, you were stuck relying on well-meaning strangers' vague left/right/straight ahead instructions – which invariably turned out to be wrong.
These days, just pull out your phone and bring up Google Maps. Likewise, the old-fashioned, drunken pub squabbles about a film or a sports team (I'm sure the first Star Wars film came out in 1975; No, I'm sure it was 1977 etc etc) are now relics of a bygone era … just "Google it"!
Of course, behemoths such as Google, Amazon and Microsoft have been gathering up all this harmless data about you for years, designing digital ads and product placements that are cleverly crafted to appeal to your particular interests and whims. Their revenue and profit growth depend upon it.
The thing is, in a sector that thrives on innovation and invention, these capabilities have become old news. Both customers and investors need something more if they're going to help catapult a company's stock price into the stratosphere.
So how do you know where to put your money?
Companies with legs, not legacy
Enter a handful of experienced fund managers in the sector, who are as much about excluding the losers as they are including the winners – those with strong growth profiles with solid fundamentals.
Jeremy Gleeson, lead fund manager of FundCalibre's Elite Rated AXA Framlington Global Technology, believes one part of the answer is to avoid the slow growth 'old guard' tech companies.
He says money spent by enterprises on maintaining legacy systems is declining, which is reflected in the revenue of firms such as IBM (down 15% in 2015), while 'new technology' spend is increasing rapidly.
In this sphere, Jeremy likes companies such as Salesforce.com, who develop cloud-based marketing and data solutions, and whose share price rose 32% in 2015 versus IBM's 14% decline, he says.
On the other hand, the tech sector can throw up more than its fair share of 'blue sky' companies at the other end of the spectrum, with grand ideas but no income and growing development costs. Steering clear of these is also a core pillar of Jeremy's strategy.
Betting, shopping and property
Meanwhile, James Thomson, fund manager of Rathbone Global Opportunities fund, also Elite Rated by FundCalibre, says his top three contributors to returns in 2015 were Betfair, Amazon and Rightmove. Rightmove and Betfair are cases in point when it comes to identifying digital disruptors outside the specialist tech sector.
James describes the property listing website, Rightmove, as "the best investment of [his] career", saying it now provides real estate agents with an average 70% of their leads and dominates the UK market.
When Betfair launched its online gambling operations in the UK, it quickly differentiated itself from competitors with innovations such as its 'Cash Out' and 'Price Rush' products, he says, both of which offer customers additional betting features.
Another fund actively looking for digitally disruptive companies outside the tech sector is Elite Rated Polar Capital Healthcare Opportunities. Its co-manager, Dr Dan Mahony, prefers to be underweight in the pharmaceuticals sector relative to the fund's benchmark index but likes to invest in businesses he believes are using technology creatively to innovate in the healthcare space.
He mentions a product called 'Invisalign', which is being used in dentistry for braces, as an example. Basically, you go to the doctor or dentist, have your teeth scanned, the scan is emailed to California, and someone 3D prints your brace and sends it back to you! A couple of weeks later, as your teeth move, you do the same thing again. There are a few regulatory issues around 3D printing in other areas of healthcare, but it will surely be used more and more in the future, Dr Dan predicts.
Is Japan the new Silicon Valley?
New kid on the block Neptune Global Technology fund, managed by Alistair Unwin, selects tech investments through a geographic lens and, in addition to the US, is interested in developments in Japan and Asia. A sentiment which is shared by Praveen Kumar, fund manager of the Baillie Gifford Japan Smaller Companies fund, which looks for high growth start-ups run by young, ambitious entrepreneurs.
Attitudes to risk taking and entrepreneurship are changing in Japan, Praveen believes, with an increasing number of founder-managed companies now coming to market. He likes companies such as Broadleaf, which has developed a cloud-based car parts ordering system to increase productivity in the automotive industry.
As always, I'd remind readers the tech sector overall tends to be extremely volatile and it's not an area where I'd personally park a significant portion of my savings. Given the diversity of developments in new and interesting areas, however, it's always worth keeping an eye on the latest trends.
START RECEIVING FREE INVESTMENT IDEAS TODAY
This post is amended from a FundCalibre blog originally published on Mindful Money, 28 January 2016.
Three investment themes for 2023
It's that time of year when everyone reaches for their crystal balls to predict what sectors could...
The most popular funds of 2022
As 2022 draws to a close, we take a look at which funds and trusts were most popular among...


HOUSTON, Texas – The Atlanta Falcons were in complete control through the first 30 minutes of Super Bowl 51 in Houston.
Tom Brady, trying to become the first quarterback to win five Super Bowls, never led his team to a point in the first quarter of his previous six Super Bowls, including his four wins.
In the 2004 Super Bowl in the same building, when the Patriots beat Carolina 32-29, it was scoreless after the first quarter in Houston's NRG Stadium.
Each team punted twice in the first quarter. The biggest highlight was Devonta Freeman's 37-yard run for Atlanta. But the Falcons didn't get another first down on that drive.
For the first time in 11 years, both teams punted on each of their first two offensive possessions in a Super Bowl.
However, the Atlanta Falcons took the lead with 12:15 left in the second quarter on a 5-yard touchdown run by Devonta Freeman.
The five-play, 71-yard drive was set up when Deion Jones forced a fumble by Patriots running back LeGarrette Blount, and the ball was recovered by Atlanta's Richard Alford.
Matt Ryan, the NFL MVP, then threw passes of 19 and 23 yards to Julio Jones, who didn't catch a pass in the first quarter.
The Falcons kept the momentum when Ryan threw a 19-yard touchdown pass less than four minutes later to tight end Austin Hooper.
Ryan, trying to become the first player to be the NFL's MVP and win a Super Bowl in the same season since 1999, is 7-of-8 passing for 115 yards.
The latest drive, after a Patriots punt, started with Ryan hitting Taylor Gabriel for 24 yards and then throwing an 18-yard pass to Julio Jones, who made a nice tip-toe catch along the sideline.
Things simply got worse for New England when Tom Brady threw an interception to Robert Alford, who ran it back 82 yards for a touchdown.
The Patriots finally started moving the ball down field, but were stopped short of a touchdown.
Instead, New England settled for a field goal to finish off the first half.
In the third quarter, Ryan threw a 6-yard touchdown pass to Tevin Coleman, capping an 85-yard drive that spanned eight plays.
At the point, the Falcons were leading the Patriots 28-3.
The Pats responded with a touchdown of their own as Tom Brady hit James White near the two yard line. However, New England ended up missing the extra point- leaving the score at 28-9.
According to Elias Sports, no team has won a Super Bowl after trailing by more than 10 points.
The Pats weren't done, after forcing a turnover, Tom Brady connected on a six yard touchdown strike to Danny Ammendola. New England went for two and converted which made it a one possession game, 28-20.
Following that, Brady led the Patriots down the field and Dion White ran in a one yard touchdown. However, the Pats had to convert a two point conversion to tie the game. Tom Brady again found Danny Ammendola for two and the game was tied at 28.
The game went to overtime and Tom Brady was spectacular. He won his 5th Super Bowl as James White scored from a yard out to win the Super Bowl 34-28.


This is just an opinion, but I think I'm not too far off: chess endings are the most difficult phase of the game. Paraphrasing from the back cover of Polgar's Chess Endgames: "'Fewer pieces, fewer problems' Endgames are proof that this is not the case". Minefields abound, proper tempo calculation is needed, apparently few players bother. This is an opportunity! If one could master, or at least get hold of the ropes, one could confidently, safely anchored in theory, steer the game to an endgame and just watch the opponent flounder and sink.
There is yet another obstacle to be surmounted. To use the theory, one must learn it first and many of the books and software out there are beyond even reasonably rated players. I think I have found the book that can get us over the hump.
Silman's Complete Endgame Course is one of the most popular endgame books nowadays and with reason. Mr. Silman is a US player, coach and an International Master. Also, he has a clear and very agreeable writing style. His Understanding Chess column in Chess Life was one of my favorites back in the 90s. His endgame course book is very unlike others in the field. Instead of starting with pawn endgames and then moving on to minor and major pieces endgames, he rather organizes his material according to player's strength. This is the key of this book. You only have to read when it's appropriate for your level and then move on as your strength increases. Why is this important? First, as said, endgames are very tricky. Starting off at pawn endgames, which are the trickiest of all, can demoralize the reader and even if he somewhat manages to go through the material, it is unlikely to help him much if his strength is not on par with what he just read. Second, lower rated players are the least likely to survive into an endgame. They will probably blast or get blasted in the opening or middlegame, and so need the least amount of knowledge. Better start modestly: If one perseveres, one can learn a little more, and then a little more; all according to what one may need most at any given time.
Chess is meant to be fun, and it is no fun when you have to go through reams of variations which do not relate to you. That's another aspect of this book. Only what is relevant is given. Other manuals and books seem to be easily waylaid by beautiful variations or artistic concepts, which ultimately have little practical value. All these are nixed here. The concept is explained and the main line plus two or three relevant variations are given; not more. So, at the end (wink) what must be remembered is very compact.
At the conclusion of each rating section there is a chapter summary and some problems for the reader to solve and check his grasp on the concepts. The last two sections do not have problems and are somewhat different than the rest. The first one is geared to master level players and explains some useful strategic ideas rather than concrete configurations. The other one gives chess endgames to enjoy from the greatest endgame players according to the author and some funny tactics & situations. An always useful further reading section is also included at the very last.
Having read the whole book, I can say that it really encourages progress. I find it hard to find anything wrong with it. The book size is a minor problem. Two or three positions are scarier than what the author reveals (these are #279, 287 and 291). Does the course really work? Effort and time wise I would say it really does. You just skip all other alternatives and get directly to the juicy parts. Performance wise, in my experience, I have better results now, but still feel I need refreshers, either by rechecking the material or looking for more exercises elsewhere, but my guess is that this would also be true for other books.
If you have been a Microsoft Office user over the last decade, you might have noticed a small purple colored program creeping in over time. It might have started, with just a menu item, then an icon in the taskbar. As of now it is still gaining ground on the cloud nearing par with the better-known applications Word and Excel. This program is OneNote.
OneNote is a notetaking application. You can think of it as a digital notebook. Remember the ones you used at school back in the day? Word already lets you input text and organize your notes, so what's the need for another such program? The beauty of OneNote is that it lets you input text anywhere on the 'paper'. You are not limited to guidelines, paragraphs or even paper size. This is a boon, because it lets you convey your creative ideas just as you want them, and just as you would using pen and paper. And that is just half of it. The other half is that when you have those notes you care about in the program, you can easily search through them with just a keystroke, or mouseclick. Oh, and it is not just text. You can also insert pictures, videos, sounds, math formulas and doodle around. If you are blessed with a handwriting that the computer likes, it can even digitize it into searchable text.
Once I began to dig deeper into it, I began to pass all my notes to it. Even if I use pen and paper initially, if I really want to keep what I have written, I promptly transcribe it to OneNote. And there is another benefit: your notes are always neat. Rearranging and deleting do not result in smudges and unsightly pages as would in physical paper.
And still more: Access. With the current shift to cloud computing, you're not condemned to have your notes locked up in your desktop's hard drive. Via Microsoft's free SkyDrive service you can save your notes to the cloud and make them available to your other various devices, such as tablets and smartphones. Doing so also lets you have multiple people work on the same notes simultaneously. A great way to collaborate. Synchronization is automatic.
So far I have only tried the iPad app on the mobile front. This one still has a long way to go, but you can edit and read your notes, even if they are displayed not quite right. Doodles and ink notes are not supported as of now and tables can only be viewed but not edited. Also, it can sometimes be slow regardless of your Internet connection speed. Too much processing going on I guess. If your device has a full-fledged browser, you can log on to your SkyDrive account and have more functionality there.
The only other downsides to OneNote as a whole that I have come across, is that there is no search-and-replace feature and that synchronization can sometimes have conflicts. For me, these are minor complaints.
There are a fair amount of alternatives to OneNote. One of the best-known is Evernote. Its main virtue is its ubiquitousness: you can get it for about any platform out there, which lets you easily consolidate what you jot down on the go with everything else. You can also include audio, photos, which are text searchable, and videos. Having said that, it only has limited functionality and doesn't compare with a full-fledged features in OneNote. There are free and paid versions of this one. Gave it a few tries, didn't like it.
I have not tried this one, but it looks very interesting. Almost as a clone of OneNote! If you hate mainstream programs, check out this one. Free for personal use.
For us, computer users, there is a group of four or five core programs that are so useful that figuratively we cannot live without. Each of us has different preferences and needs which reflect in what we use. The rise of mobile devices and applications has shifted the focus towards smartphones and tablets, but there are still some persons, like me, that prefer the larger screens and more computing power; that is why my core programs are still run on a PC. I thought to review them for anyone out there who is listening, because these might prove useful to them too. I'll start in this post with my macro program.
I'm a man of routine. No, I don't mean that I'm boring, but that I lean towards a more ordered life. Part of that is trying to eliminate wasted time. Doing the same things over and over again can take big chunks of your life. Now, that's boring. Even with computers, most people still have to fire up the machine, open the browser, open the several favorite webpages one reads daily, navigate within those, write several mails with perhaps identical headings if the job requires it, and once in a while process a large number of items, such as files or filenames, cell contents in Excel, or the like. Wouldn't it be wonderful if one could achieve this and more with just the press of a button and more efficiently to boot? Well, this is what a macro program can do for you.
My favorite macro program from the start, bar none, is Quick Macros. Quick Macros is developed by Gintaras Didzgalvis and as of this writing is in version 2.3.3.7. I first began to use it back in version 1. It is lightweight yet powerful. For my purposes I only use a sliver of its potential, yet, I doubt that anyone who gets deep into it will be disappointed. It is accessible, requiring from you just some basic programming concepts.
Writing a macro script is fast. You can write something useful within a minute or something more elaborate if the task requires it. The language is clear and intuitive. The code itself is color-coded for easy navigation. For those less inclined to programming, there is a handy integrated recorder that lets you record the precise mouse movements and keystrokes you wish to replicate in the future. With just that you are good to go. There are several triggers for launching a macro. You can trigger one with a keystroke, mouse click or movement, window event, and even by time. The macro runs until it finishes, gets stuck, or if you decide to end it prematurely by pressing the pause key.
If the need arises, you can export/import the macro files or even make a .exe file to run on another computer that doesn't have quick macros.
The documentation is a bit abstruse, but by looking at the examples one can figure out what each function does most of the time. There is a forum at its homepage with active participation of Gintaras. Answers to most questions can be found there, which sometimes take the form of "how can I…?".
Probably the biggest caveat is that you cannot run two macros simultaneously. Yet, most of the time one can think of a workaround. Also, do not assume that the macros work instantaneously. They do take their time following the script you give them, which can take a while for longer ones. This is normal and still a much better deal than doing the tasks by hand. If I remember rightly, it takes a 10th of a second to pass from instruction to instruction; what takes a longer while is waiting for events to happen, such as waiting for a window to be created.
So far I have used it to launch Internet radio at a certain time, install media players and other programs quickly, resize several windows to a preferred layout, automate futures order entry, play chess with just my voice (more on that on a future post), fill in databases easier, kill pop-up windows, and even edit other macro codes within the same QM screen.
The most obvious alternative is Autohotkey which is free. I tried it a couple of times, but I found the learning curve much much too steep and the flow of writing and launching more difficult. Nonetheless, it also has great capabilities. AHK once could pull off something that I just couldn't get QM to perform.
This is paid software ($40), but you can try it out free for 30 days. If you're in need of something of this sort, it is well worth it.
During my childhood, my family had a cherry tree in the garden and I liked it much. These are also very popular over at Japan. They believe them very beautiful and frankly, they are. When they bloom, they are a sight like no other. The pink petals, which are soft and delicate, blow with the wind in spring.
It may come to a surprise that these petals can be collected and added to foodstuffs such as pastries. Accordingly, it is not a too much of a stretch to have them in your tea. Tokyo Printemps (the name will surely vary from merchant to merchant) is a cherry flavored Japanese green tea sencha. I gave it a try after giving it a sniff at my local provider. The green leaves are mostly whole and so are the dried petals. Most of the flavoring appears to come from added essential oils. The aroma, unsurprisingly, is cherry like; but when I open the packet at home it gives me an unusual sensation: I feel as if I'm unwrapping a present from a loving daughter. True!
For a pot you need about the usual quantity of leaf. The taste is also like cherries, but not so much like the fruit itself. The grassiness of the green tea is subdued or almost entirely cloaked. Care must be taken to not oversteep it as the flavor can get overly intense resulting in what I would call cherry sting.
I believe this would make a very good tea for a morning tea party.
The downside, at least for me, is my own palate which has been accustomed to years of artificial cherry flavoring in medicines and the like. When I drink it, I am sometimes reminded of those; and medicine is not something you want to be reminded of when drinking your pleasant tea. The case is also similar to that of the Soleil Levant from Hediard, which instead of tasting like oranges, tastes like children's medicine. I know, I know: it is the medicine that tastes like oranges, not the other way around. With the Printemps, you can avoid this by not steeping it too much. Oh, there is also the lesser a complaint in that it is not fit for re-steeping.
If you're a cherry fan or at least have a passable interest, do ask for this tea, but if you're like me you might probably want to pass on this one and just watch the flowers.