text:
<?
// Copyright 2014 Tera Insights, LLC. All Rights Reserved.

function DATETIME() {
    $methods = [];
    $constructors = [];
    $functions = [];
    $bin_operators = [];
    $libraries = [];

    $system_headers = [ 'cinttypes', 'ctime', 'cstdio' ];

    $system_headers[] = 'boost/date_time/posix_time/posix_time.hpp';
    $system_headers[] = 'boost/date_time/gregorian/gregorian.hpp';
    $libraries[] = 'boost_date_time';

    $globalContent = "";
?>

class DATETIME {
private:
    static constexpr const int32_t TM_YEAR_OFFSET = 1900;

    static constexpr const int32_t INVALID_DTIME = -1;

    static constexpr const int32_t S_PER_MIN = 60;
    static constexpr const int32_t S_PER_HR = S_PER_MIN * 60;
    static constexpr const int32_t S_PER_DAY = S_PER_HR * 24;

    // Each month is encoded in 5 bits, with january in the least significant
    // bits
    static constexpr const uint64_t DAYS_PER_MONTH = 0x0ffbfefffdff7f9f;
    static constexpr const uint8_t DPM_MASK = 0x1F;
    static constexpr const uint8_t DPM_BITS = 5;

    int32_t ctime;

    void Set(
        int16_t     year,
        int8_t      month,
        int8_t      day,
        int8_t      hour,
        int8_t      minute,
        int8_t      second
    );

public:
<?  $constructors[] = [[], true];  ?>
    constexpr DATETIME(void);

<?  $constructors[] = [['BASE::INT'], true];  ?>
    constexpr DATETIME(const int32_t _ctime);

<?  $constructors[] = [['BASE::NULL'], true];  ?>
    constexpr DATETIME(const GrokitNull & nullval);

<?  $constructors[] = [['BASE::SMALLINT', 'BASE::BYTE', 'BASE::BYTE', 'BASE::BYTE', 'BASE::BYTE', 'BASE::BYTE'], true];  ?>
    DATETIME(
        int16_t    year,
        int8_t     month,
        int8_t     day,
        int8_t     hour,
        int8_t     minute,
        int8_t     second
    );

<?  $constructors[] = [['BASE::STRING_LITERAL'], true]; ?>
    DATETIME(const char * str);

    // Accessors for portions of the date
<?  $methods[] = ['Year', [], 'BASE::SMALLINT', true];  ?>
    int16_t Year(void) const;
<?  $methods[] = ['Month', [], 'BASE::BYTE', true];  ?>
    int8_t  Month(void) const;
<?  $methods[] = ['Day', [], 'BASE::BYTE', true];  ?>
    int8_t  Day(void) const;
<?  $methods[] = ['Hour', [], 'BASE::BYTE', true];  ?>
    int8_t  Hour(void) const;
<?  $methods[] = ['Minute', [], 'BASE::BYTE', true];  ?>
    int8_t  Minute(void) const;
<?  $methods[] = ['Second', [], 'BASE::BYTE', true];  ?>
    int8_t  Second(void) const;

<?  $methods[] = ['DayOfWeek', [], 'BASE::BYTE', true];  ?>
    int8_t  DayOfWeek(void) const;
<?  $methods[] = ['DayOfYear', [], 'BASE::SMALLINT', true];  ?>
    int16_t DayOfYear(void) const;

    // Returns the number of days in the date's month
<?  $methods[] = ['DaysInMonth', [], 'BASE::BYTE', true]; ?>
    int8_t  DaysInMonth(void) const;

<?  $methods[] = ['AsDays', [], 'BASE::INT', true] ?>
    constexpr int32_t AsDays(void) const;

<?  $methods[] = ['AsHours', [], 'BASE::INT', true]; ?>
    constexpr int32_t AsHours(void) const;

<?  $methods[] = ['AsMinutes', [], 'BASE::INT', true];  ?>
    constexpr int32_t AsMinutes(void) const;

<?  $methods[] = ['AsSeconds', [], 'BASE::INT', true];  ?>
    constexpr int32_t AsSeconds(void) const;

<?  $methods[] = ['IsValid', [], 'BASE::BOOL', true];  ?>
    constexpr bool IsValid(void) const;
<?  $methods[] = ['IsNull', [], 'BASE::BOOL', true];  ?>
    constexpr bool IsNull(void) const;

<?  $methods[] = ['IsDST', [], 'BASE::BOOL', true]; ?>
    bool IsDST(void) const;

    // Comparisons
<?  $bin_operators[] = '==';  ?>
    constexpr bool operator ==(const DATETIME & o) const;
<?  $bin_operators[] = '!=';  ?>
    constexpr bool operator !=(const DATETIME & o) const;
<?  $bin_operators[] = '<';  ?>
    constexpr bool operator <(const DATETIME & o) const;
<?  $bin_operators[] = '>';  ?>
    constexpr bool operator >(const DATETIME & o) const;
<?  $bin_operators[] = '<=';  ?>
    constexpr bool operator <=(const DATETIME & o) const;
<?  $bin_operators[] = '>=';  ?>
    constexpr bool operator >=(const DATETIME & o) const;

    constexpr bool operator ==(const int32_t secs) const;
    constexpr bool operator !=(const int32_t secs) const;
    constexpr bool operator <(const int32_t secs) const;
    constexpr bool operator >(const int32_t secs) const;
    constexpr bool operator <=(const int32_t secs) const;
    constexpr bool operator >=(const int32_t secs) const;

    // Adding / subtracting dates
    constexpr DATETIME operator +(const int32_t secs) const;
    constexpr DATETIME operator -(const int32_t secs) const;
    constexpr int32_t operator -(const DATETIME & o) const;

    void FromString( const char * str );
    int ToString( char * buffer ) const;

    void FromJson( const Json::Value & );
    void ToJson( Json::Value & ) const;

    // Static Members //

    static constexpr int8_t DaysInMonth(int16_t year, int8_t month);

private:
    static constexpr int8_t GetDPM(int8_t month) {
        return (DAYS_PER_MONTH >> (month * DPM_BITS)) & DPM_MASK;
    }
};

inline
constexpr int8_t DATETIME::DaysInMonth(int16_t year, int8_t month) {
    return ((month != 1) || (year % 4 != 0) || (year % 100 == 0 && year % 400 != 0)) ?
        GetDPM(month) : GetDPM(month) + 1;
}

inline
void DATETIME :: Set( int16_t year, int8_t month, int8_t day, int8_t hour, int8_t minute, int8_t second ) {
    tm dtime;
    dtime.tm_year = year - TM_YEAR_OFFSET;
    dtime.tm_mon = month;
    dtime.tm_mday = day;
    dtime.tm_hour = hour;
    dtime.tm_min = minute;
    dtime.tm_sec = second;
    dtime.tm_isdst = -1;

    time_t c_time = mktime( &dtime );
    ctime = static_cast<int32_t>(c_time);
}

inline constexpr
DATETIME :: DATETIME(void) : ctime(INVALID_DTIME)
{ }

inline constexpr
DATETIME :: DATETIME(const int32_t _ctime) : ctime(_ctime)
{ }

inline constexpr
DATETIME :: DATETIME(const GrokitNull & nullval) : ctime(INVALID_DTIME)
{ }

inline
DATETIME :: DATETIME(const char * str)
{
    this->FromString(str);
}

inline
DATETIME :: DATETIME( int16_t year, int8_t month, int8_t day, int8_t hour, int8_t minute, int8_t second ) {
    Set(year, month, day, hour, minute, second);
}

inline
int16_t DATETIME :: Year(void) const {
    auto temp = boost::posix_time::from_time_t(static_cast<time_t>(ctime));
    return temp.date().year();
}

inline
int8_t DATETIME :: Month(void) const {
    auto temp = boost::posix_time::from_time_t(static_cast<time_t>(ctime));
    return temp.date().month();
}

inline
int8_t DATETIME :: Day(void) const {
    auto temp = boost::posix_time::from_time_t(static_cast<time_t>(ctime));
    return temp.date().day();
}

inline
int8_t DATETIME :: Hour(void) const {
    auto temp = boost::posix_time::from_time_t(static_cast<time_t>(ctime));
    return temp.time_of_day().hours();
}

inline
int8_t DATETIME :: Minute(void) const {
    auto temp = boost::posix_time::from_time_t(static_cast<time_t>(ctime));
    return temp.time_of_day().minutes();
}

inline
int8_t DATETIME :: Second(void) const {
    auto temp = boost::posix_time::from_time_t(static_cast<time_t>(ctime));
    return temp.time_of_day().seconds();
}

inline
int8_t DATETIME :: DayOfWeek(void) const {
    auto temp = boost::posix_time::from_time_t(static_cast<time_t>(ctime));
    return temp.date().day_of_week();
}

inline
int16_t DATETIME :: DayOfYear(void) const {
    auto temp = boost::posix_time::from_time_t(static_cast<time_t>(ctime));
    return temp.date().day_of_year();
}

inline
int8_t DATETIME :: DaysInMonth(void) const {
    auto temp = boost::posix_time::from_time_t(static_cast<time_t>(ctime));
    auto ymd = temp.date().year_month_day();
    return DATETIME::DaysInMonth(ymd.year, ymd.month - 1);
}

inline
bool DATETIME :: IsDST(void) const {
    tm dtime;
    time_t tmp = static_cast<time_t>(ctime);
    gmtime_r(&tmp, &dtime);
    mktime(&dtime);

    return dtime.tm_isdst;
}

inline constexpr
int32_t DATETIME :: AsDays(void) const {
    return ctime / S_PER_DAY;
}

inline constexpr
int32_t DATETIME :: AsHours(void) const {
    return ctime / S_PER_HR;
}

inline constexpr
int32_t DATETIME :: AsMinutes(void) const {
    return ctime / S_PER_MIN;
}

inline constexpr
int32_t DATETIME :: AsSeconds(void) const {
    return ctime;
}

inline constexpr
bool DATETIME :: IsValid(void) const {
    return ctime >= 0;
}

inline constexpr
bool DATETIME :: IsNull(void) const {
    return ctime < 0; //>
}

inline constexpr
bool DATETIME :: operator ==(const DATETIME & o) const {
    return ctime == o.ctime;
}

inline constexpr
bool DATETIME :: operator !=(const DATETIME & o) const {
    return ctime != o.ctime;
}

inline constexpr
bool DATETIME :: operator <(const DATETIME & o) const {
    return ctime < o.ctime;
}

inline constexpr
bool DATETIME :: operator >(const DATETIME & o) const {
    return ctime > o.ctime;
}

inline constexpr
bool DATETIME :: operator <=(const DATETIME & o) const {
    return ctime <= o.ctime;
}

inline constexpr
bool DATETIME :: operator >=(const DATETIME & o) const {
    return ctime >= o.ctime;
}

inline constexpr
bool DATETIME :: operator ==(const int32_t secs) const {
    return ctime == secs;
}

inline constexpr
bool DATETIME :: operator !=(const int32_t secs) const {
    return ctime != secs;
}

inline constexpr
bool DATETIME :: operator <(const int32_t secs) const {
    return ctime < secs;
}

inline constexpr
bool DATETIME :: operator >(const int32_t secs) const {
    return ctime > secs;
}

inline constexpr
bool DATETIME :: operator <=(const int32_t secs) const {
    return ctime <= secs;
}

inline constexpr
bool DATETIME :: operator >=(const int32_t secs) const {
    return ctime >= secs;
}

inline constexpr
DATETIME DATETIME :: operator +(const int32_t secs) const {
    return DATETIME(ctime + secs);
}

inline constexpr
DATETIME DATETIME :: operator -(const int32_t secs) const {
    return DATETIME(ctime - secs);
}

inline constexpr
int32_t DATETIME :: operator -( const DATETIME & o ) const {
    return ctime - o.ctime;
}

inline
void DATETIME :: FromString( const char * str ) {
    using namespace boost::posix_time;

/*
    short year;
    char month;
    char day;
    char hour;
    char minute;
    char second;

    sscanf(str, "%hd-%hhd-%hhd %hhd:%hhd:%hhd", &year, &month, &day, &hour, &minute, &second);
*/

    ptime posixTime = time_from_string(str);
    ptime epoch(boost::gregorian::date(1970, 1, 1));
    time_duration::sec_type x = (posixTime - epoch).total_seconds();

    ctime = int32_t(x);
}

inline
int DATETIME :: ToString( char * buffer ) const {
    tm dtime;
    time_t tmp = static_cast<time_t>(ctime);
    gmtime_r(&tmp, &dtime);

    const char * format = "%d-%02d-%02d %02d:%02d:%02d";

    return 1+sprintf(buffer, format,
        dtime.tm_year + TM_YEAR_OFFSET,
        dtime.tm_mon + 1,
        dtime.tm_mday,
        dtime.tm_hour,
        dtime.tm_min,
        dtime.tm_sec
    );
}

inline
void DATETIME :: FromJson( const Json::Value & src ) {
    this->FromString(src.asCString());
}

inline
void DATETIME :: ToJson( Json::Value & dest ) const {
    char buffer[24];
    this->ToString(buffer);
    dest = buffer;
}

<?  ob_start(); ?>

inline
void FromString( @type & date, const char * buffer ) {
    date.FromString( buffer );
}

inline
int ToString( const @type & date, char * buffer ) {
    return date.ToString(buffer);
}

inline
int64_t ClusterValue(const @type& x){
    // Cast to uint first so that the result is not sign extended
    uint32_t tmp = x.AsSeconds();
    return tmp;
}

inline
void FromJson( const Json::Value & src, @type & dest ) {
    dest.FromJson(src);
}

inline
void ToJson( const @type & src, Json::Value & dest ) {
    src.ToJson(dest);
}

// Hash function
<?  $functions[] = ['Hash', ['@type'], 'BASE::BIGINT', true, true ]; ?>
inline uint64_t Hash( const @type val ) {
    return val.AsSeconds();
}

<?  $functions[] = ['IsNull', ['@type'], 'BASE::BOOL', true, true ]; ?>
inline
bool IsNull( const @type & d ) {
    return d.IsNull();
}

#ifdef _HAS_STD_HASH
<?  $system_headers[] = 'functional'; ?>
// C++11 STL-compliant hash struct specialization

namespace std {

template <>
class hash<@type> {
public:
    size_t operator () (const @type& key) const {
        return Hash(key);
    }
};

}
#endif // _HAS_STD_HASH

<?  $globalContent .= ob_get_clean(); ?>

<?
    return [
        'kind'              => 'TYPE',
        'complex'           => false,
        'system_headers'    => $system_headers,
        'user_headers'      => [ 'Config.h' ],
        'binary_operators'  => $bin_operators,
        'global_content'    => $globalContent,
        'constructors'      => $constructors,
        'methods'           => $methods,
        'functions'         => $functions,
        'libraries'         => $libraries,
        'properties'        => [ 'clusterable' ],
        'describe_json'     => DescribeJson('datetime', DescribeJsonStatic(['format' => 'YYYY-MM-DD HH:mm:ss'])),
        'extras'            => [ 'size.bytes' => 4 ],
    ];
}

// DATETIME + INT = DATETIME
declareOperator( '+', [ 'base::DATETIME', 'base::INT' ],
    function($args, $targs = []) {
        $dateType = lookupType('base::DATETIME');
        $intType = lookupType('base::INT');

        return [
            'kind'      => 'OPERATOR',
            'input'     => [ $dateType, $intType ],
            'result'    => $dateType,
            'deterministic' => true,
        ];
    }
);

// DATETIME - INT = DATETIME
declareOperator( '-', [ 'base::DATETIME', 'base::INT' ],
    function($args, $targs = []) {
        $dateType = lookupType('base::DATETIME');
        $intType = lookupType('base::INT');

        return [
            'kind'      => 'OPERATOR',
            'input'     => [ $dateType, $intType ],
            'result'    => $dateType,
            'deterministic' => true,
        ];
    }
);

// DATETIME - DATETIME = INT
declareOperator( '-', [ 'base::DATETIME', 'base::DATETIME' ],
    function($args, $targs = []) {
        $dateType = lookupType('base::DATETIME');
        $intType = lookupType('base::INT');

        return [
            'kind'      => 'OPERATOR',
            'input'     => [ $dateType, $dateType ],
            'result'    => $intType,
            'deterministic' => true,
        ];
    }
);

foreach( [ '==', '!=', '<', '>', '<=', '>=' ] as $op ) {
    declareOperator( $op, [ 'base::DATETIME', 'base::INT' ],
        function($args, $targs = []) {
            $dateType = lookupType('base::DATETIME');
            $intType = lookupType('base::INT');
            $boolType = lookupType('base::BOOL');

            return [
                'kind'      => 'OPERATOR',
                'input'     => [ $dateType, $intType ],
                'result'    => $boolType,
                'deterministic' => true,
            ];
        }
    );
}
?>

redpajama_set_name: RedPajamaGithub


text:
Округ Десото () је округ у америчкој савезној држави Мисисипи.

Демографија
По попису из 2010. године број становника је 161.252, што је 54.053 (50,4%) становника више него 2000. године.

Референце

Литература

Види још
 Округ
 Спискови округа САД

Спољашње везе

 -{United States Census Bureau}-

Окрузи Мисисипија

redpajama_set_name: RedPajamaWikipedia


text:
Labour advice amounts to 'ultra-nanny state'
Commenting on the launch of the Welsh Labour government's 'Bed times without the battles and shopping without the stropping' campaign, Welsh Conservative Shadow Minister for Education Angela Burns AM, said:
"I'm entirely unconvinced that it's the government's job to tell parents how to bring up their children.
"Taxpayers will rightly question the resources going into a Labour scheme that bears all the hallmarks of an ultra-nanny state.
"On Universal Children's Day we should spend time considering the support available for young people and highlighting how we can improve young lives.
"It's not a day for the government to highlight 'shopping without stropping'.
"There are many first class charities and organisations that already provide exemplary advice for parents.
"The Welsh government should be working with them – and ensuring all parents are aware of the support on offer."
Assembly News

redpajama_set_name: RedPajamaCommonCrawl


text:
Sunglass Hut rolls out digital capability to improve in-store experiences
Sunglasses retailer invests in digitised merchandising solution as part of efforts to better personalise physical store locations
Sunglasses retailer, Sunglass Hut, says the decision to implement new technology to localise merchandise recommendations in-store is about improving the bricks-and-mortar experience through digital capability.
The retail group, which has more than 2700 locations globally and is part of the Luxottica group, has invested in One Door's Merchandising Cloud application and rolling out the solution across its Australia and New Zealand operations.
The vendor said the platform combines product and promotional visual merchandising with store information in order to provide a digitised version of each store location and its features. Using the app, a retailer can capture and tap into a digital model of store attributes, floorplans, fixtures and resources and come up with an interactive merchandising plan based on these elements.
Sunglass Hut marketing and ecommerce director, Kate Grech, said the in-store experience is integral to high-fashion accessories and sunglasses in particular because products are chosen to reflect the personality of the consumer. Using the platform was about localising and personalising product mixes in-store so they feel more custom-made to that location. They can then analyse status and results based on each SKU, fixture, store and campaign.
"As our business continues to grow, One Door's Merchandising Cloud will assist us in delivering a consistent in-store experience and keep pace with rapidly changing customer needs," Grech said.
CEO of One Door, EY Snowden, said the platform was being implemented and supported by its global team. The vendor rebranded to One Door from RBM Technologies in December after a three-year transformation program that included bringing in a new leadership team, global sales and support investment, rearchitecting its SaaS-based application, and a brand overhaul.
"High-fashion accessory retailers like Sunglass Hut need to offer customers a wide range of fresh and relevant products, and then enable store associates to maximise their time helping customers to make great choices," he said.
"This will let Sunglass Hut collaborate and accelerate the delivery of localised products, even while simplifying the associate's job so they can focus on delivering great, personalised, in-store experiences."
Other One Door customers globally include O2, Vodafone, Cox Communications and
Read more 10 brands using digital to create great in-store experiences
Follow CMO on Twitter: @CMOAustralia, take part in the CMO conversation on LinkedIn: CMO ANZ, join us on Facebook: https://www.facebook.com/CMOAustralia, or check us out on Google+:google.com/+CmoAu
How digital transformation opened a new customer segment for Kennards Hire
CMO's top 8 martech stories for the week - 16 February 2017
Tags: retaildigital strategymarketing technology


text:
Samuel Velebný (9. listopadu 1918, Pukanec – 11. října 1991, Košice) byl evangelický kněz, pronásledovaný komunistickým režimem. V roce 1962 byl odsouzen a následně i uvězněn na 21 měsíců ve Valdicích.

Životopis
Samuel Velebný pocházel z rodiny místního kloboučníka jako nejmladší ze tří dětí. Nejprve navštěvoval měšťanskou školu v rodišti, později studoval na evangelickém gymnáziu v Prešově (do roku 1939). Citlivé vnímání života ho přivedlo k rozhodnutí zasvětit svůj život službě lidem, kteří zápasili s těžkými, nejednou existenčními problémy. Proto si po střední škole zvolil studium evangelické teologie v Bratislavě, kde působil do roku 1943.

Po ukončení školy se nakrátko stal evangelickým kaplanem v Brezně a do roku 1947 působil jako kaplan při seniorovi Bakossovi. Zde ho zastihlo Slovenské národní povstání a s ním spojené smutné osudy mnoha rodin, kterým se snažil pomoci různým způsobem. Riskantní bylo zejména křtění Židů, aby se zachránili před transportem do koncentračního tábora. Jako farář pomáhal uvězněným v té době, ale i lidem v době povstání, aby mohli přežít. Vedle toho se věnoval mládeži, kterou si získával svou srdečností a otevřeností. V listopadu 1947 byl ustanoven farářem do gemerské obce Sirk. Zde získal první negativní zkušenosti se Státní bezpečností, která ho zajistila, protože v kázání vyjádřil vděčnost za osvobození obce Bohu a ne sovětské armádě. 14. ledna 1949 ho zajistily orgány VB, držely ve vazbě v Revúci, ale na soudu v Rimavské Sobotě byl 29. ledna zproštěn obžaloby a propuštěn na svobodu. V květnu 1949 ho ale odvedli do tábora nucené práce v Novákách. Tato záležitost souvisí se zajištěním orgány VB v lednu 1949; na službách Božích poděkoval Pánu Bohu za dar svobody a nepoděkoval se za osvobození Sovětské armádě.

Tato okolnost byla důvodem pro odchod ze Sirku v roce 1949 do další gemerské obce Vlachovo, kde byl řádně zvoleným farářem od 3. listopadu 1949. Ve Vlachově pocítil omezující působení státních orgánů vůči církvi, protože nastalo období intenzivního prosazování ateistické ideologie vládnoucí komunistické strany (KSČ), jejímž cílem bylo vychovat pokrokově smýšlejícího člověka, nezatíženého minulostí. Problémy s ideologickým režimem se ještě více stupňovaly vzhledem ke vzniku skupiny KVAP. Skupinu KVAP tvořili čtyři faráři z Gemera, a to Bedřich Kasanický z Dobšiné, Samuel Velebný z Vlachova, Jan Antal z Betliara a Jan Paulův z Gemerské Polomy. Tito faráři se nejprve setkávali v kruhu svých rodin, ale postupně se jejich setkání změnila i na místa teologického a duchovního růstu. Po dohodě se rozhodli vypracovat společný soubor kázání. Časem však VB jejich setkání vyhodnotila jako protistátní. Členové skupiny KVAP byli obviňováni z různých činů, které měly ohrozit socialistický režim.

Nakonec byli tito čtyři faráři obžalováni a zatčeni. Jelikož se během výslechu pod nátlakem přiznali, začaly proti nim soudní spory v Košicích, které trvaly od 16. do 18. května 1962. Samuel Velebný byl odsouzen na 20 měsíců odnětí svobody nepodmíněně. Jemu a Janu Antalovi soud zároveň udělil zákaz vykonávat povolání duchovního a jakékoli náboženské funkce na dobu 2 let. U všech obžalovaných propadly ve prospěch státu všechny do úschovy vzaté písemnosti při vyšetřování, kázání, knihy a pod. Později v případě Samuela Velebného, Josefa Bedřicha Kasanického a Jana Antala zasedal Nejvyšší soud v Praze na základě odvolání krajského státního zástupce. Jmenovaní dostali přísnější a definitivní tresty. Samuel Velebný dostal tři roky odnětí svobody. Délka zákazu vykonávat farářskou činnost zůstala nezměněna. Odsouzení se ocitli ve známé věznici ve Valdicích, kde komunistický režim do roku 1963 koncentroval politické vězně odsouzené na principu náboženské diskriminace. Samuel Velebný při broušení českého křišťálu a jiných pracích přežil 21 měsíců až do propuštění. Relativně nízká sazba trestů ve spojení s praxí odpuštění poloviny trestů v případě dobrého chování vězně napomohly k tomu, že obviněné propustili dříve. U Samuela Velebného to bylo 17. září 1963. Podmíněné propuštění z vězení znamenalo, že Samuel Velebný tři roky nesměl vykonávat povolání. Když k tomu připočteme dva roky zákazu činnosti, které potvrdil i Nejvyšší soud v Praze, bylo to pět let.

Po návratu z vězení byl Samuel Velebný zaměstnán jako stavební dělník v podniku Stavoindustria, který budoval provoz chemického závodu v nedaleké Hnúšti. "Přiměřeně rozsudku" se opět zapojil i do práce v církevním sboru Klenovec, což mohlo být chápáno jako nedodržení podmínky. Účinkoval totiž jako kantor na službách Božích uprostřed týdne, nazývaných i večerní. Koncem prosince 1966 dostal Samuel Velebný státní souhlas k výkonu farářského povolání a GBU mu nabídl místo ve Vyšné Kamenici s nástupem od ledna 1967. I nadále po propuštění za ním občas přicházel agent StB. Soudní procesy s faráři se už však nekonaly. V květnu 1968 požádal Nejvyšší soud v Praze o rehabilitaci. Když se jí nedočkal, psal žádost na Krajský soud v Košicích. I ten ji však zamítl. Svou aktivní službu ukončil Samuel Velebný poslední listopadovou neděli roku 1987 a rozloučil se s Vyšnou Kamenicí po 21 letech působení. Poslední roky života strávil s rodinou v Košicích. V roce 1990 dostal oficiální poděkování z rukou evangelického biskupa Uhorskaia za to, že statečně trpěl pro Pána Boha a jeho církev. Samuel Velebný zemřel 11. října 1991, necelý měsíc před dovršením 73. roku života.

Literatura
 Kdo byl kdo v historii Banské Bystrice
 Evanjelici v dejinách slovenskej kultúry
 J.G. Farár Juráš a "KVAP" ve Východoslovenských novinách, 4. augusta 1962

 Froňková, Viola. SAMUEL VELEBNÝ- obhajca Kristovej pravdy v Evanjelický Posol Spod Tatier, 26. říjen 2011, str.18
 Froňková, Viola. Pre všetkých v rodine bol láskavým a vzácnym človekom v Evanjelický Posol Spod Tatier, 26. říjen 2011, str.19
 Havíra, Ján. Novodobé dejinné križovatky našej cirkvi v Evanjelický Posol Spod Tatier, č.6/1993, str.44
 Zajden, Michal. DVE SMUTNÉ výročia (I. část) v Evangelický Posel Spod Tater, 16. květen 2012, str.8
 Zajden, Michal. DVE SMUTNÉ výročia (II. část) v Evangelický Posel Spod Tater, 23. květen 2012, str.8
 Zajden, Michal. DVE SMUTNÉ výročia (III. část) v Evangelický Posel Spod Tater, 30. květen 2012, str.8

Externí odkazy
 https://web.archive.org/web/20170112075238/http://banskabystrica.kniznice.net/katalog/clpr54.htm

Slovenští protestantští duchovní
Političtí vězni komunistického režimu v Československu
Narození v roce 1918
Narození 9. listopadu
Narození v Pukanci
Úmrtí v Košicích
Úmrtí v roce 1991
Úmrtí 11. října
Muži


Counting Birds is a nonfiction picture book that tells the story behind bird counting and how everyday kids can take part in protecting bird species near and far.
By Yolen, Jane; Stemple, Heidi E. Y.
From Cleopatra to Lizzie Borden, meet 26 of history's most notorious women. Each bad girl has a rotten reputation, but there are two sides to every tale. Each chapter ends with comic panels featuring caricatures of the authors discussing the women. Illustrations.
15 September 2015 by Charlesbridge Publishing,U.S.
By Yolen, Jane; Stemple, Heidi E. Y.; Ashby, Gil; Stemple, Heidi Elisabet y.


text:
Kirby Smart praises Justin Fields' maturity in handling playing time
Andrew Olson | 2 years ago
In the Week 9 36-17 win over Florida, Georgia fans were pleased with the outcome, but some were surprised with how the Bulldogs got to that result. True freshman QB Justin Fields, the top quarterback recruit of the 2018 class, did not get into the Florida game, which was especially surprising considering UF used the bye week to work its freshman quarterback into the gameplan.
After Saturday's SEC East-clinching win over Kentucky, UGA coach Kirby Smart praised Fields for the way he went about his business after the Florida game in preparing for the UK contest.
"Justin, I told our guys, here's a guy who didn't play a snap last week," Smart told reporters, per Dawgs247. "You didn't hear anything out of him. All he did is come out and work this week. I thought he had his best practice of the year on Tuesday and he had a good practice on Wednesday. I told him, I said, 'Man, this guy's practicing. He's buying in, he's positive, cheering on the sideline and he goes in and plays well."
Fields played six snaps against Kentucky, running the ball each time for a total of 26 yards.
Smart indicated he would like to get Fields more playing time in passing situations.
"I just wish we could get to the point that we could play him a little more because he's getting better," the coach said. "He does a lot of good things. And he doesn't just have to run the ball. The situations today dictated that and he did a great job of doing it."
A University of Florida alumnus, Andrew has been writing about SEC football for Saturday Down South since 2015. Follow him on Twitter @ByAndrewOlson.
redsox2004 2 years ago
When Fromm loses his job in the spring, will he stay or will he go?
My teams are perennial rollercoasters of emotion 2 years ago
They'll string him along like yall did Jalen to where it's too late to transfer and he's played in 5 games before he knows it so he can't redshirt.


, we all picks the very best choices with best resolution exclusively for you all, and this images is usually among images collections in this ideal graphics gallery regarding Management Experience Resume. I'm hoping you will enjoy it.
submitted simply by keepupdated at 2018-07-26 22:50:45. To discover almost all images within Management Experience Resume graphics gallery make sure you stick to this particular website link.
Always remembers to look up the next image gallery, which also includes the Management Experience Resume On Retail Store Manager Resume Best Retail Resume 0d Archives – Resume photo showed above. If you have any thoughts, questions or just want to say hello to other people, please do not hestitate to submit your opinion/ideas via the following comment form.


text:
About U.S. Naval Institute Essay Contests
CNO Naval History
Emerging & Disruptive Technologies
Enlisted Prize
NPS Foundation/U.S. Naval Institute
Naval Mine Warfare
The Proceedings Podcast
Contact Proceedings
With more and more programs added to S-4, he Secretaries of the Army, Navy, and Air Force (right to left) reminded the Senate Armed Services Committee of the danger that the bill could become so expensive that it could force reductions in other areas.
U.S. NAVY (TERRY COSGROVE)
Points of Interest: The Fight for S-4
By Tom Philpott
View Issue
In late February, on a bipartisan vote of 91 to 8, the Senate passed the Soldiers', Sailors', Airmen's, and Marines' Bill of Rights Act of 1999, the most robust military compensation package in a generation.
The "core package" of the bill, also known as S-4, includes:
Join the U.S. Naval Institute to gain access to this article and other articles published in Proceedings since 1874. Members receive this valuable benefit and so much more.
If you are already a Member, please log in to gain access, and thank you for your Membership.
More Stories From This Author View Biography
Digital Proceedings content made possible by a gift from CAPT Roger Ekman, USN (Ret.)


We thought the rock is a bit lichen covered, this will clean up to be a great route. The crux comes high up before the anchors on the light colored rock. This arching climbs offers great position and unique moves. It is probably the best of the three on the Upper Deck.
Start in the obvious corner that is left of the pinyon pine. Place some gear and/or clip a head. Mantle up and clip bolts as they arc left. Make a few moves straight up over a bulge and piece together the thin crux before the end!
This is far right up the broken ledges on Upper Deck ledge. It is best if you 4th class it to the medium pinyon pine tree where slings are visible. You can top-rope from the ledge of the Upper Deck.
1 or 2 small nuts, a small piece for the start.


text:
\section{Introduction}

Deep Neural Networks (DNNs) have achieved substantial improvements in several fields of machine learning. However, since DNNs have a highly nonlinear and non-convex structure, the fundamental principles behind their remarkable performance is still an open problem. Therefore, advances in this field largely depend on heuristic approaches. One of the most prominent techniques to boost the generalization performance of DNNs is regularizing layer weights so that the network can fit a function that performs well on unseen test data. Even though weight decay, i.e., penalizing the $\ell_2^2$-norm of the layer weights, is commonly employed as a regularization technique in practice, recently, it has been shown that $\ell_2$-path regularizer \cite{neyshabur_pathnorm}, i.e., the sum over all paths in the network of the squared product over all weights in the path, achieves further empirical gains \cite{neyshabur_pathsgd}. Therefore, in this paper, we investigate the underlying mechanisms behind path regularized DNNs through the lens of convex optimization.

\begin{figure*}[h]
        \centering
            \centering
            \includegraphics[width=1\textwidth,height=0.4\textwidth]{Figures/path_complete.pdf}
            \caption{\textbf{(Left):} Parallel ReLU network in \eqref{eq:parallel_arc} with $K$ sub-networks and three layers ($L=3$)  \textbf{(Right):} Path regularization for a three-layer network.}\label{fig:subnetworks
    \end{figure*}

\section{Parallel Neural Networks}
Although DNNs are highly complex architectures due to the composition of multiple nonlinear functions, their parameters are often trained via simple first order gradient based algorithms, e.g., Gradient Descent (GD) \cite{brutzkus_localminima}. However, since such algorithms only rely on local gradient of the objective function, they tend to fail to globally optimize the objective in some cases \cite{shalev2017failures,goodfellow2016deep}. Similarly, \cite{ge2017onehidden,shamir2018spurious} showed that these pathological cases also apply to stochastic algorithms such as Stochastic GD (SGD). They further proved some of these pathological cases can be avoided by increasing the number of trainable parameters, i.e., operating in an overparameterized regime. However, \cite{anandkumar2016saddle} reported the existence of more complicated cases, where SGD/GD usually fails. Therefore, training DNNs is an extremely challenging optimization problem \cite{dasgupta1995complexity,blum1989threenode,bartlett1999hardness}.

To circumvent training issues, recent studies focused on proposing new NN models benefiting from impacts of overparameterization on the optimization landscape \cite{brutzkus_overparameterized_linear,du2018overparameterized,arora2018overparameterization,neyshabur2018overparameterization}. As an example, \cite{wang2021parallel,ergen2021revealing, ergen2021deep,zhang2019multibranch,haeffele2017global} considered a new architecture by combining multiple standard NNs, termed as sub-networks, in parallel. Evidences in \cite{ergen2021revealing, ergen2021deep,zhang2019multibranch,haeffele2017global,zagoruyko2016wideresidual,veit2016ensembleresidual} showed that this way of combining NNs yields an optimization landscape that has fewer local minima and/or saddle points so that SGD/GD generally converges to a global minimum with a remarkable generalization performance. Therefore, most of the recently proposed NN-based architectures that achieve a state-of-the-art performance in practice, e.g., SqueezeNet \cite{iandola2016squeezenet}, Inception \cite{szegedy2017inception}, Xception \cite{chollet2017xception}, and ResNext \cite{xie2017aggregatedresidual}, are in this form.

\subsection{Notation and preliminaries}
Throughout the paper, we denote matrices and vectors as uppercase and lowercase bold letters, respectively. For vectors and matrices, we use subscripts to denote a certain column/element. As an example, $w_{lk j_{l-1} j_{l}}$ denotes the $j_{l-1} j_{l}^{th}$ entry of the matrix $\vec{W}_{lk}$. We use $\vec{I}_k$ and $\vec{0}$ (or $\vec{1}$) to denote the identity matrix of size $ k \times k$ and a vector/matrix of zeros (or ones) with appropriate sizes. We use $[n]$ for the set of integers from $1$ to $n$. We use $\|\cdot\|_2$ and $\|\cdot \|_{F}$ to represent the Euclidean and Frobenius norms, respectively. Additionally, we denote the unit $\ell_p$ ball as $\mathcal{B}_p:=\{\vec{u} \in \mathbb{R}^d\, :\,\|\vec{u}\|_p\le 1\}$. We also use $\mathbbm{1}[x\geq0]$ and $\relu{x}=\max\{x,0\}$ to denote the 0-1 valued indicator and ReLU, respectively.

In this paper, we particularly consider a parallel ReLU network with $K$ sub-networks and each sub-network is an $L$-layer ReLU network (see Figure \ref{fig:subnetworks}) with layer weights $\vec{W}_{lk} \in \mathbb{R}^{m_{l-1} \times m_l}$, $\forall l \in [L-1]$ and $\vec{w}_{Lk} \in \mathbb{R}^{m_{L-1}}$, where $m_0=d$, $m_L=1$\footnote{In the paper, we analyze scalar outputs,  however, our derivations extend to vector output networks as shown in Appendix \ref{sec:supp_vectorout}.}, and $m_l$ denotes the number of neurons in the $l^{th}$ hidden layer. Then, given a data matrix ${\bf X} \in \mathbb{R}^{n \times d}$, the output of the network is as follows
\begin{align} \label{eq:parallel_arc}
    f_{\theta}({\bf X}):=\sum_{k=1}^K \relu{\relu{{\bf X} \vec{W}_{1k}} \ldots \vec{W}_{(L-1)k} }\vec{w}_{Lk},
\end{align}
where we compactly denote the parameters as $\theta:=\bigcup_{k}\{\vec{W}_{lk}\}_{l=1}^L$ with the parameter space as $\Theta$ and each sub-network represents a standard deep ReLU network.
\begin{rems}\label{rem:parallel_extensions}
We note that most of the commonly used neural networks architectures in practice can be classified as special cases of our parallel ReLU network, e.g., standard NNs and ResNets \cite{resnet} see Appendix \ref{sec:supp_parallel_networks} for details.
\end{rems}

\begin{table*}
  \caption{Complexity comparison with prior works ($n$: $\#$ of data samples, $d$: feature dimension, $m_l$: $\#$ of hidden neurons in layer $l$, $\epsilon$: approximation accuracy, $r$: rank of the data, $\kappa \ll r$: chosen according to \eqref{eq:approximation_ineq} such that $\epsilon$ accuracy is achieved) }
  \label{tab:complexity}
  \centering
    \resizebox{2\columnwidth}{!}{
\begin{tabular}[b]{|c|c|c|c|c|}
\hline
& \textbf{Loss function} & \textbf{2-layer complexity}& \textbf{$L$-layer complexity}  & \textbf{Globally optimal}\\  \hline
{\cite{arora2018understanding}}& Convex loss & $\mathcal{O}\left(2^{m_1} n^{dm_1}poly(n,d,m_1)\right)$  & -  &  \ \cmark(Brute-force)  \\ \hline
{\cite{goel2021hardness}} & $\ell_2$-loss  &  $2^{\mathcal{O}(\frac{m_1^5}{\epsilon^2})}  poly(n,d)$  & -  &  \ \xmark (NP-hard)\\ \hline
{\cite{froese2021computational}} & $\ell_p$-loss  & $\mathcal{O}\left(2^{m_1} n^{dm_1}poly(n,d,m_1)\right)$      & -  &    \  \cmark(Brute-force)   \\ \hline
{\cite{pilanci2020neural}} & Convex loss    &    $\mathcal{O}(n^r poly(d,r) )$  &  - &    \  \cmark (Convex, exact)   \\ \hline
\textbf{Ours} & Convex loss & $\mathcal{O}( n^r poly(d,r) )$  & $\mathcal{O}\left(n^{r\prod_{j=1}^{L-2}m_j}poly(d,r,\prod_{j=1}^{L-2}m_l) \right)$  &  \cmark  (Convex, exact) \\
\hline
\textbf{Ours} & Convex loss & $\mathcal{O}( n^\kappa poly(d,\kappa) )$  & $\mathcal{O}\left(n^{\kappa\prod_{j=1}^{L-2}m_j}poly(d,\kappa,\prod_{j=1}^{L-2}m_l) \right)$  &  \cmark  (Convex, $\epsilon$-opt) \\
\hline
\end{tabular}}%
\end{table*}

\subsection{Our Contributions}
\begin{itemize}[leftmargin=*]

\item  We prove that training the path regularized parallel ReLU networks \eqref{eq:parallel_arc} is equivalent to a single convex optimization problem that can be solved in polynomial-time by standard convex solvers. To the best of our knowledge, this is the first work proving polynomial-time trainability for regularized ReLU networks with more than one hidden non-linear layers (see Table \ref{tab:complexity}) without any assumption. Therefore, we generalize the two-layer results \cite{pilanci2020neural} to multiple nonlinear layer without any strict assumptions unlike \cite{ergen2021deep} and a much broader class of NN architectures including ResNets.

\item As already observed by \cite{pilanci2020neural, ergen2021deep}, regularized deep ReLU network training problems require exponential-time complexity when the data matrix is full rank, which is unavoidable. However, we develop a training algorithm which has fully polynomial-time complexity for all datasets and prove global optimality guarantees in Theorem \ref{theo:lowrank_approx}. \textbf{To the best of our knowledge, this is the first convex and fully polynomial-time training algorithm for deep networks with global optimality guarantees.}

\item We show that the equivalent convex problem is regularized by a group norm regularization where grouping effect is among sub-networks.Therefore the equivalent convex formulation reveals an implicit regularization that promotes group sparsity among sub-networks.

\item We derive a closed-form mapping between the parameters of the non-convex parallel ReLU networks and its equivalent convex problem in Proposition \ref{prop:weight_construction}. Therefore, instead of solving the challenging non-convex problem, one can globally solve the equivalent convex problem and then construct an optimal solution to the original non-convex network architecture via our closed-form mapping.

\end{itemize}

\subsection{Overview of Our Results}
 Given a data matrix ${\bf X} \in \mathbb{R}^{n \times d}$ and a label vector $\vec{y} \in \mathbb{R}^n$, we consider the following regularized training problem
\begin{align}
    p_L^*:=\min_{\theta \in \Theta} \genloss{ f_{\theta}({\bf X}),\vec{y} }+\beta  \mathcal{R}(\theta) \,, \label{eq:problem_statement}
\end{align}
where $\Theta$ is the parameter space, $\genloss{\cdot,\cdot}$ is an arbitrary convex loss function, $\mathcal{R}(\cdot)$ represents the regularization on the network weights, and $\beta>0$ is the regularization coefficient.

For the rest of the paper, we focus on a conventional scalar output regression/classification framework with an arbitrary loss function, e.g., squared loss, cross entropy or hinge loss. We also note that our derivations can be straightforwardly extended to vector outputs networks as proven in Appendix \ref{sec:supp_vectorout}. More importantly, instead of standard weight decay regularization, we use $\ell_2$-path regularizer studied in \cite{neyshabur_pathnorm,neyshabur_pathsgd}, which is defined as
\begin{align*}
    \mathcal{R}(\theta):=\sum_{k=1}^{K} \sqrt{\sum_{j_1,j_2,\ldots, j_{L}}\left( \|\vec{w}_{1kj_1}\|_2^2 \prod_{l=2}^{L} w_{lk j_{l-1} j_{l}}^2\right)} ,
\end{align*}
where $w_{lk j_{l-1} j_{l}}$ is the $j_{l-1} j_{l}^{th}$ entry of $\vec{W}_{lk}$. The above regularizer sums the square of all the parameters along each possible path from input to output of each sub-network $k$ (see Figure \ref{fig:subnetworks}) and then take the squared root of the summation. Therefore, we penalize each path in each sub-network and then group them based on the sub-network index $k$.

We now propose a rescaling for the layer weights such that the problem in \eqref{eq:problem_statement} can be equivalently cast as a group $\ell_1$ regularized optimization problem as follows.

\begin{lems}\label{lemma:scaling_deep_overview}
The following problems are equivalent \footnote{All the proofs are presented in the supplementary file.}:
{\medmuskip=0mu
\thinmuskip=0mu
\thickmuskip=0mu
\begin{align*}
\min_{\theta \in \Theta}  \genloss{ f_{\theta}({\bf X}),\vec{y} }+ \beta\mathcal{R}(\theta)
=
\min_{{\theta \in \Theta_s} } \genloss{ f_{\theta}({\bf X}),\vec{y} }+ \beta \sum_{k=1}^K \|\vec{w}_{Lk}\|_2,
\end{align*}}
where $\vec{w}_{Lk} \in \mathbb{R}^{m_{L-1}}$ are the last layer weights of each sub-network $k$, and $\Theta_s:=\{\theta \in \Theta:  \sum_{j_1,j_2,\ldots, j_{L-2}}\left( \|\vec{w}_{1j_1}\|_2^2 \prod_{l=2}^{L-1} w_{l j_{l-1} j_{l}}^2\right) \leq 1, \forall j_{L-1}\}$ denotes the parameter space after rescaling.
\end{lems}

The major advantage of the form in Lemma \ref{lemma:scaling_deep_overview} is that we can derive a dual problem with respect to the output layer weights $\vec{w}_{Lk}$ and then characterize the set of optimal layer weights via the KKT conditions and the prior works on $\ell_1$ regularization in infinite dimensional spaces \cite{rosset2007}. Thus, we first apply the rescaling in Lemma \ref{lemma:scaling_deep_overview} and then take the dual with respect to the output weights $\vec{w}_{Lk}$. In order to characterize the hidden layer weights, we then change the order of minimization for the hidden layer weights and the maximization for the dual parameter to get the following dual problem\footnote{We present the details in Appendix \ref{sec:supp_dualproblem}.}
\begin{align}\label{eq:dual_overview}
        &p_L^*\geq d_L^* :=\max_{\vec{\vec{v}}} - \mathcal{L}^*(\vec{v})  \\ \nonumber
        &\text{ s.t. } \max_{\theta \in \Theta_s} \left\| \vec{v}^T \relu{\relu{{\bf X} \vec{W}_{1}}\ldots \vec{W}_{(L-1)}} \right\|_2\leq \beta,
\end{align}
where $\mathcal{L}^*$ is the Fenchel conjugate function of the loss function $\mathcal{L}$, which is defined as \cite{boyd_convex}
\begin{align*}
\mathcal{L}^*(\vec{v}) := \max_{\vec{z}} \vec{z}^T \vec{v} - \genloss{\vec{z},\vec{y}}.
\end{align*}
The dual problem in \eqref{eq:dual_overview} is critical for our derivations since it provides us with an analytic perspective to characterize a set of optimal hidden layer weights for the non-convex neural network in \eqref{eq:parallel_arc}. To do so, we first show that strong duality holds for the non-convex training problem in Lemma \ref{lemma:scaling_deep_overview}, i.e., $p_L^*=d_L^*$. Then, based on the exact dual problem in \eqref{eq:dual_overview}, we propose an equivalent analytic description for the optimal hidden layer weights via the KKT conditions.

We note that strong duality for ReLU networks has already been proved by previous studies \cite{wang2021parallel,ergen2021deep,pilanci2020neural,ergen2020journal,zhang2019multibranch,bach2017breaking}, however, this is the first work providing an exact characterization for path regularized deep ReLU networks via convex duality.

\section{Parallel networks with three-layer layers}
Here, we consider a three-layer parallel network with $K$ sub-networks, which is a special case of \eqref{eq:parallel_arc} when $L=3$. Thus, we have the following training problem
{\medmuskip=0mu
\thinmuskip=0mu
\thickmuskip=0mu
\begin{align}\label{eq:newmodel_primal1}
        p_3^*=\min_{\theta \in \Theta} \mathcal{L}\left(f_{\theta}({\bf X}) ,\vec{y}\right)+\beta \sum_{k=1}^{K} \sqrt{\sum_{j_1,j_2} \|\vec{w}_{1kj_1}\|_2^2 w_{2kj_1j_2}^2 w_{3kj_2}^2}.
\end{align}}
where $\Theta= \{(\vec{W}_{1k},\vec{W}_{2k},\vec{w}_{3k}): \vec{W}_{1k}\in \mathbb{R}^{d \times m_1}, \vec{W}_{2k}\in \mathbb{R}^{m_1 \times m_2},\vec{w}_{3k}\in \mathbb{R}^{m_2} \}$. By Lemma \ref{lemma:scaling_deep_overview}, we have
\begin{align}\label{eq:newmodel_primal2}
           p_3^*=\min_{\theta \in \Theta_s} \mathcal{L}\left( f_{\theta}({\bf X}) ,\vec{y}\right)+\beta \sum_{k=1}^K\|\vec{w}_{3k}\|_2.
\end{align}
Then, taking the dual of \eqref{eq:newmodel_primal2} with respect to the output layer weights $\vec{w}_{3k} \in \mathbb{R}^K$ and then changing the order of the minimization for $\{\vec{W}_{1k},\vec{W}_{2k}\}$ and the maximization for the dual variable $\vec{v}$ yields
\begin{align}\label{eq:newmodel_dual1}
 &p_3^* \geq d_3^*:= \max_{\vec{v}} -\mathcal{L}^*(\vec{v})\\ \nonumber
 &\text{ s.t. } \max_{\theta \in \Theta_s}  \left\|\vec{v}^T \relu{\relu{{\bf X} \vec{W}_{1}}\vec{W}_{2}}\right\|_2\leq \beta.
\end{align}
Here, we remark that even though \eqref{eq:newmodel_primal1} is convex with respect to the output layer weights (because the architecture in \ref{eq:parallel_arc} is linear in $\vec{w}_{Lk}$), we have a duality gap, i.e., $p_3^* \geq d_3^*$, due to the change in the order of min-max (i.e. $\min_x\max_y f(x,y)\geq \max_y \min_x f(x,y)$). Therefore, we first show that strong duality holds in this case, i.e., $p_3^* \geq d_3^*$ as detailed in Appendix \ref{sec:supp_mainproof}. We then introduce an equivalent representation for the ReLU activation as follows.

\begin{figure*}[t]
        \centering
            \centering
            \includegraphics[width=\textwidth]{Figures/hyperplane_arrangements_horizontal.pdf}
            \caption{Illustration of possible hyperplane arrangements that determine the diagonal matrices $\mathbf{D}_i$ for a toy dataset. }\label{fig:hyperplanes}
    \end{figure*}
Since ReLU is a function that masks the negative entries of inputs, we have the following equivalent representation
\begin{align} \label{eq:relu_alternative}
\relu{\relu{{\bf X} \vec{W}_1}\vec{w}_{2}}&=\relu{\sum_{j_1=1}^{m_1} \relu{{\bf X} \vec{w}_{1j_1}}w_{2j_1}} \nonumber\\
&=\relu{\sum_{j_1=1}^{m_1} \relu{{\bf X} \vec{w}_{j_1}}\mathcal{I}_{j_1}} \nonumber \\
&=\mathbf{D}_{2}\sum_{j_1=1}^{m_1}\mathcal{I}_{j_1} \mathbf{D}_{1j}{\bf X} \vec{w}_{j_1},
\end{align}
where $\vec{w}_{j_1}=\vert w_{2j_1}\vert\vec{w}_{1j_1}$, $\mathcal{I}_{j_1} =\mathrm{sign}(w_{2j_1})\in \{-1,+1\}$, and we use the following alternative representation for ReLU (see Figure \ref{fig:hyperplanes} for a two dimensional visualization)
{\medmuskip=0mu
\thinmuskip=0mu
\thickmuskip=0mu
\begin{align*}
    \relu{{\bf X}\vec{w}}=\mathbf{D} {\bf X} \vec{w} \iff \begin{split}
        &\mathbf{D}{\bf X} \vec{w} \geq 0\\
        &(\vec{I}_n-\mathbf{D}){\bf X} \vec{w} \leq 0
    \end{split}\iff (2\mathbf{D}-\vec{I}_n){\bf X} \vec{w} \geq 0,
\end{align*}}
where $\mathbf{D} \in \mathbb{R}^{n \times n}$ is a diagonal matrix of zeros and ones, i.e., $\mathbf{D}_{ii} \in \{0,1\}$. Therefore, we first enumerate all possible signs and diagonal matrices for the first and second ReLU layers and denote them as $\mathcal{I}_{j_1}$, $\mathbf{D}_{1ij_1} $ and $\mathbf{D}_{2l}$ respectively, where $ j_1 \in [m_1] ,  i \in [P_1],  l \in [P_2]$. Then, we convert the non-convex dual constraints in \eqref{eq:newmodel_dual1} to a convex constraint using the fixed diagonal matrices $\mathbf{D}_{1ij_1} $, $\mathbf{D}_{2l}$ and signs $\mathcal{I}_{j_1}$.

Using the representation in \eqref{eq:relu_alternative}, we then take the dual of \eqref{eq:newmodel_dual1} to obtain the convex bidual of the primal problem \eqref{eq:newmodel_primal1} as detailed in the theorem below.

\begin{theos} \label{theo:main_theorem}
The non-convex training problem in \eqref{eq:newmodel_primal1} can be cast as the following convex program
\begin{align}
\label{eq:newmodel_final}
  &\min_{\vec{z},\vec{z}^\prime\in \mathcal{C}} \mathcal{L}\left(\tilde{{\bf X}}  (\vec{z}-\vec{z}^\prime),\vec{y}   \right)+\frac{\beta}{\sqrt{m_2}}  (\|\vec{z}\|_{F,1}+\|\vec{z}^\prime\|_{F,1})
\end{align}
where $\|\cdot\|_{F,1}$ denotes a $d \times m_1$ dimensional group Frobenius norm operator such that given a vector $\vec{u} \in \mathbb{R}^{d m_1 P}$, $\|\vec{u}\|_{F,1}:= \sum_{i=1}^P \|\vec{U}_{i} \|_F$, where $\vec{U}_{i} \in \mathbb{R}^{d \times m_1}$ are reshaped partitions of $\vec{u}$. Moreover, the convex set $\mathcal{C}$ is defined as
\begin{align*}
     &\mathcal{C}:=\{\vec{z}:\vec{z}_{ij_1l}^{{s}} \in\mathcal{C}_{il}^{s}, \forall i \in [P_1],l \in [P_2],s \in [M]\}\\ &\mathcal{C}_{il}^{s}:=\bigg\{\{\vec{w}_{j_1} \}_{j_1} :(2\mathbf{D}_{2l} -\vec{I}_n)\sum_{j_1=1}^{m_1} \mathcal{I}_{ij_1l}^{s}\mathbf{D}_{1ij_1}{\bf X} \vec{w}_{j_1} \geq 0,\\
    &(2\mathbf{D}_{1ij_1} -\vec{I}_n){\bf X} \vec{w}_{j_1}\geq 0,  \forall j_1 \in [m_1]\bigg\}
\end{align*}
where $\mathcal{I}_{ij_1l}^{s} \in \{+1,-1\}$, $M=2^{m_1}$, and $\vec{z},\vec{z}^\prime  \in \mathbb{R}^{ d m_1 M P_1 P_2}$ are constructed by stacking $\vec{z}_{ij_1l}^{{s}},\vec{z}_{ij_1l}^{{s^\prime}}    \in \mathbb{R}^d$, $\forall i \in [P_1], l\in[P_2], j_1 \in [m_1], s\in [M]$, respectively. Also, the effective data matrix $\tilde{{\bf X}} \in \mathbb{R}^{n \times d m_1 M P_1 P_2}$ is defined as
\begin{align*}
  &\tilde{{\bf X}}:= \vec{I}_{M}\otimes\tilde{{\bf X}}_s,\\
  &\tilde{{\bf X}}_s:= \begin{bmatrix}\mathbf{D}_{21}  \mathbf{D}_{111}{\bf X} \ldots  \mathbf{D}_{2l}  \mathbf{D}_{1ij_1}{\bf X} \ldots \mathbf{D}_{2P_2}  \mathbf{D}_{1P_1 m_1} {\bf X}\end{bmatrix}.
\end{align*}
\end{theos}
We next derive a closed-form mapping between the convex program \eqref{eq:newmodel_final} and the non-convex architecture \eqref{eq:newmodel_primal1}.
\begin{props} \label{prop:weight_construction}
An optimal solution to the non-convex parallel network training problem in \eqref{eq:newmodel_primal1}, denoted as $\{\vec{W}_{1k}^*,\vec{w}_{2k}^*,w_{3k}^*\}_{k=1}^K$, can be recovered from the optimal solution to the convex program in \eqref{eq:newmodel_final}, i.e., $\{\vec{z}^*,\vec{z}^{\prime^*}\}$ via a closed-form mapping. Therefore, we prove a mapping between the parameters of the parallel network in Figure \ref{fig:subnetworks} and its convex equivalent.
\end{props}

Next, we prove that the convex program in \eqref{eq:newmodel_final} can be globally optimized with a polynomial-time complexity given ${\bf X}$ has fixed rank, i.e., $\mathrm{rank}({\bf X})=r< \min\{n,d\}$.

\begin{props} \label{prop:complexity}
 Given a data matrix such that $\mathrm{rank}({\bf X})=r< \min\{n,d\}$, the convex program in \eqref{eq:newmodel_final} can be globally optimized via standard convex solvers with  $\mathcal{O}(d^3 m_1^3 m_2^3 2^{3 (m_1+1)m_2} n^{3(m_1+1)r})$ complexity, which is a polynomial-time complexity in terms of $n,d$..
\end{props}

Below, we show that the complexity analysis in Proposition \ref{prop:complexity} extends to arbitrarily deep parallel networks.
\begin{cors} \label{cor:complexity_deep}
The same analysis can be readily applied to arbitrarily deep networks. Therefore,  given $\mathrm{rank}({\bf X})=r< \min\{n,d\}$, we prove that $L$-layer architectures can be globally optimized
with $\mathcal{O}\left(d^3 \left(\prod_{j=1}^{L-2}m_j^3\right) 2^{3 \sum_{j=1}^{L-1}m_j}n^{3r \left(1+\sum_{l=1}^{L-2}\prod_{j=1}^{l}m_j\right)} \right)$, which is polynomial in $n,d$.
\end{cors}

\subsection{Fully polynomial-time training for arbitrary data  }
Based on the analysis above, exponential complexity is unavoidable for deep networks when the data matrix is full rank, i.e., $\mbox{rank}({\bf X})= \min\{n,d\}$. Thus, we propose a low rank approximation to the model in \eqref{eq:newmodel_primal1}. We first denote the rank-$r$ approximation of ${\bf X}$ as $\hat{{\bf X}}_r$ such that $\|{\bf X}-\hat{{\bf X}}_r\|_2 \leq \sigma_{r+1}$, where $\sigma_{r+1}$ represents the $(r+1)^{th}$ largest singular value of ${\bf X}$. Then, we have the following result.

\begin{theos}\label{theo:lowrank_approx}
Given an $R$-Lipschitz convex loss function $\genloss{\cdot,\vec{y}}$, the regularized training problem
{\medmuskip=0mu
\thinmuskip=0mu
\thickmuskip=0mu
\begin{align}\label{eq:twolayer_objective_generic_lowrank}
   p_3^*=\min_{\theta \in \Theta} \genloss{f_{\theta}({\bf X}),\vec{y}}+\beta \sum_{k=1}^{K} \sqrt{\sum_{j_1,j_2} \|\vec{w}_{1kj}\|_2^2 w_{2kj_1j_2}^2 w_{3kj_2}^2}\,,
\end{align}}
can be solved using the data matrix $\hat{{\bf X}}_r$ to achieve the following optimality guarantee
\begin{align} \label{eq:approximation_ineq}
    p_3^* \leq p_r \leq p_3^*  \left(1+\frac{\sqrt{m_1m_2}R \sigma_{r+1}}{\beta} \right)^2,
\end{align}
where $p_r$ denotes the objective value achieved by the parameters trained using $\hat{{\bf X}}_r$.
\end{theos}

\begin{figure*}[ht]
\centering
\captionsetup[subfigure]{oneside,margin={1cm,0cm}}
	\begin{subfigure}[t]{0.45\textwidth}
	\centering
	\includegraphics[width=.9\textwidth, height=0.7\textwidth]{Figures/approximation_err2.pdf}
	\caption{Objective value ($\downarrow$)\centering} \label{fig:rank_error}
\end{subfigure} \hspace*{\fill}
	\begin{subfigure}[t]{0.45\textwidth}
	\centering
	\includegraphics[width=.9\textwidth, height=0.7\textwidth]{Figures/approximation_p2.pdf}
	\caption{$\#$ of hyperplane arrangements ($P$)\centering} \label{fig:rank_p}
\end{subfigure} \hspace*{\fill}
\caption{Verification of Theorem \ref{theo:lowrank_approx} and Remark \ref{rem:rank_approx}. We train a parallel network using the convex program in Theorem \ref{theo:main_theorem} with $\ell_2$ loss on a toy dataset with $n=15$, $\beta=0.1$, $m_2=1$, and the low-rank approximation $r= \lfloor \frac{d}{2} \rfloor$. To obtain a low-rank model, we first sample a data matrix from a standard normal Gaussian distribution and then set $\sigma_{r+1}=\ldots=\sigma_d=1$.}\label{fig:low_rank}
\end{figure*}

\begin{rems} \label{rem:rank_approx}
Theorem \ref{theo:main_theorem} and \ref{theo:lowrank_approx} imply that for a given arbitrary rank data matrix ${\bf X}$, the regularized training problem in \eqref{eq:newmodel_primal1} can be approximately solved by convex solvers to achieve a worst-case approximation $p_3^*\left(1+\frac{R R\sigma_{r+1}}{\beta}\right)^2$ with complexity $\mathcal{O}(d^3 m_1^3 2^{3 (m_1+1)} n^{3(m_1+1)r})$, where $r \ll \min\{n,d\}$. Therefore, even for full rank data matrices where the complexity is exponential in $n$ or $d$, one can approximately solve the convex program in \eqref{eq:newmodel_final} in polynomial-time. Moreover, we remark that the approximation error proved in Theorem \ref{theo:lowrank_approx} can be arbitrarily small for practically relevant problems. As an example, consider a parallel network training problem with $\ell_2$ loss function, then the upperbound becomes $(1+\frac{\sqrt{m_1m_2}\sigma_{r+1}}{\beta})^2$, which is typically close to one due to fast decaying singular values in practice (see Figure \ref{fig:low_rank}).
\end{rems}

\subsection{Representational power: Two versus three layers}

Here, we provide a complete explanation for the representational power of three-layer networks by comparing with the two-layer results in \cite{pilanci2020neural}. We first note that three-layer networks have substantially higher expressive power due to the non-convex interactions
between hidden layers as detailed in \cite{zhu_3layer,3layer_meanfield}. Furthermore, \cite{belilovsky2019greedy} show that layerwise training of three-layer networks can achieve comparable performance to deeper models, e.g., VGG-11, on Imagenet. There exist several studies analyzing two-layer networks, however, despite their empirical success, a full theoretical understanding and interpretable description of three-layer networks is still lacking in the literature. In this work, we provide a complete characterization for three-layer networks through the lens of convex optimization theory. To understand their expressive power, we compare our convex program for three-layer networks in \eqref{eq:newmodel_final} with its two-layer counterpart in \cite{pilanci2020neural}.

\cite{pilanci2020neural} analyzes two-layer networks with one ReLU layer, so that the data matrix ${\bf X}$ is multiplied with a single diagonal matrix (or hyperplane arrangement) $\mathbf{D}_i$. Thus, the effective data matrix is in the form of $\tilde{{\bf X}}_s=[\mathbf{D}_1 {\bf X} \,\ldots \,\mathbf{D}_P {\bf X}]$. However, since our convex program in \eqref{eq:newmodel_final} has two nonlinear ReLU layers, the composition of these two-layer can generate substantially more complex features via locally linear variables $\{\vec{w}_{ij_1l}^{s}\}$ which multiplies the $d$-dimensional blocks of the columns of the new data matrix $\tilde{{\bf X}}_s$ in Theorem \ref{theo:main_theorem}. Although this seems to be similar to the features in \cite{ergen2021deep}, here, we have $2^{m_1}$ variables for each linear region unlike \cite{ergen2021deep} with 2 variables per linear region. Therefore, as a by product of our analysis, we show that path regularized training problems induce a richer high dimensional convex models than the classical weight decay regularized training problems. Moreover, in the light of our results, a path regularized deep ReLU network can be precisely viewed as a high-dimensional feature selection method, where features are selected via group sparsity inducing regularization in \eqref{eq:newmodel_final}. Hence, we exactly describe the impact of having one more ReLU layer and its contribution to the representational power of the network.

\begin{table*}
  \caption{Training objective of a three-layer parallel network trained with non-convex SGD (5 independent initialization trials) on a toy dataset with $(n,d,m_1,m_2, \beta,\text{batch size})=(5,2,3,1,0.002,5)$, where the convex program in \eqref{eq:newmodel_final} are solved via the interior point solvers in CVX/CVXPY.}
\label{tab:synthetic}
  \centering
  \resizebox{2\columnwidth}{!}{
  \begin{tabular}{l|lllll|l}    \toprule
  \hspace{1.5cm} \textbf{Method} &   \multicolumn{5}{c|}{\textbf{Non-convex SGD}}   &  \multicolumn{1}{c}{\textbf{Convex(Ours)}}
 \\ \midrule
       & \textbf{Run \#1} & \textbf{Run \#2}    & \textbf{Run \#3}   & \textbf{Run \#4}   & \textbf{Run \#5}   &  \\
    \midrule
    \multirow{2}{*}{$\bm{K=5}$} Training objective &  \multicolumn{1}{c}{$0.0221$}  & \multicolumn{1}{c}{$0.0239$}  & \multicolumn{1}{c}{$0.0031$}  & \multicolumn{1}{c}{$0.0027$} & \multicolumn{1}{c}{$0.0027$}  & \multicolumn{1}{c}{$\mathbf{0.0007}$} \\
     \hspace{2cm}Time(s) &  \multicolumn{1}{c}{$11.62$}  & \multicolumn{1}{c}{$11.62$} & \multicolumn{1}{c}{$11.62$} & \multicolumn{1}{c}{$11.62$} & \multicolumn{1}{c}{$11.62$}  & \multicolumn{1}{c}{$\mathbf{4.947}$} \\
    \bottomrule
    \multirow{2}{*}{$\bm{K=20}$} Training objective &  \multicolumn{1}{c}{$0.0010$}  & \multicolumn{1}{c}{$0.0027$}  & \multicolumn{1}{c}{$0.0009$}  & \multicolumn{1}{c}{$0.0010$} & \multicolumn{1}{c}{$0.0027$}  & \multicolumn{1}{c}{$\mathbf{0.0007}$} \\
     \hspace{2cm}Time(s) &  \multicolumn{1}{c}{$44.37$}  & \multicolumn{1}{c}{$44.37$} & \multicolumn{1}{c}{$44.37$} & \multicolumn{1}{c}{$11.55$} & \multicolumn{1}{c}{$44.37$}  & \multicolumn{1}{c}{$\mathbf{4.947}$} \\
    \bottomrule
    \multirow{2}{*}{$\bm{K=40}$} Training objective &  \multicolumn{1}{c}{$0.0008$}  & \multicolumn{1}{c}{$0.0009$}  & \multicolumn{1}{c}{$0.0009$}  & \multicolumn{1}{c}{$0.0009$} & \multicolumn{1}{c}{$0.0008$}  & \multicolumn{1}{c}{$\mathbf{0.0007}$} \\
     \hspace{2cm}Time(s) &  \multicolumn{1}{c}{$91.87$}  & \multicolumn{1}{c}{$91.87$} & \multicolumn{1}{c}{$91.87$} & \multicolumn{1}{c}{$91.87$} & \multicolumn{1}{c}{$91.87$}  & \multicolumn{1}{c}{$\mathbf{4.947}$} \\
    \bottomrule
  \end{tabular} }
\end{table*}

\begin{figure*}[t]
        \centering
        \begin{subfigure}[b]{0.45\textwidth}
            \centering
         \includegraphics[width=.9\textwidth,height=.7\textwidth]{Figures/acc_cifar10_path.png}
            \caption{CIFAR-10}\label{fig:cifar}
        \end{subfigure}        \hfill
        \begin{subfigure}[b]{0.45\textwidth}
            \centering
          \includegraphics[width=.9\textwidth,height=.7\textwidth]{Figures/acc_fmnist_path.png}
            \caption{Fashion-MNIST}\label{fig:fmnist}
        \end{subfigure}
	\caption{Accuracy of a three-layer architecture trained using the non-convex formulation \eqref{eq:newmodel_primal1} and the proposed convex program \eqref{eq:newmodel_final}, where we use \textbf{(a)} CIFAR-10 with $(n,d,m_1,m_2,K,\beta,\text{batch size})=(5\text{x} 10^4,3072,100,1,40,10^{-3},10^{3})$ and \textbf{(b)} Fashion-MNIST with $(n,d,m_1,m_2,K,\beta,\text{batch size})=(6\text{x} 10^4,784,100,1,40,10^{-3},10^3)$. We note that the convex model is trained using \textbf{(a)} SGD and \textbf{(b)} Adam. }\label{fig:cifar_mnist}

    \end{figure*}

\section{Experiments}

Here\footnote{Details on the experiments can be found in Appendix \ref{sec:supp_numerical}.}, we conduct experiments corroborating the theory.

\textbf{Low rank model in Theorem \ref{theo:lowrank_approx}:}
To validate our claims, we generate a synthetic dataset as follows. We first randomly generate a set of layer weights for a parallel ReLU network with $K=5$ sub-networks by sampling from a standard normal distribution. We then obtain the labels as $\vec{y}=\sum_k\relu{\relu{{\bf X} \vec{W}_{1k}}\vec{W}_{2k}}\vec{w}_{3k}+0.1\bm{\epsilon}$, where $\bm{\epsilon} \sim N(\vec{0},\vec{I}_n)$. To promote a low rank structure in the data, we first sample a matrix from the standard normal distribution and then set $\sigma_{r+1}=\ldots=\sigma_d=1$. We consider a regression framework with $\ell_2$ loss and $(n,r,\beta,m_1,m_2)=(15,d/2,0.1,1,1)$ and present the numerical results in Figure \ref{fig:low_rank}. Here, we observe that the low rank approximation of the objective $p_r$ is closer to $p_3^*$ than the worst-case upper-bound predicted by Theorem \ref{theo:lowrank_approx}. However, in Figure \ref{fig:rank_p}, the low rank approximation provides a significant reduction in the number of hyperplane arrangements, and therefore in the complexity of solving the convex program.

\textbf{Toy dataset:} We consider a regression task on a toy dataset with $5$ samples and $2$ features, i.e., $(n,d)=(5,2)$. To generate the dataset, we forward propagate i.i.d. samples from a multivariate standard normal distribution, i.e., $\vec{x}_i \sim \mathcal{N}(\vec{0},\vec{I}_d)$, through a parallel network with $3$ layers, $5$ sub-networks, and $3$ neurons, i.e., $(L,K,m_1,m_2)=(3,5,3,1)$. We then train the parallel network in \eqref{eq:newmodel_primal1} on this toy dataset using both our convex program \eqref{eq:newmodel_final} and non-convex SGD. We provide the training objective and wall-clock time in Table \ref{tab:synthetic}, where we particularly include $5$ different initialization trials for SGD. This experiment shows that when the number of sub-networks $K$ is small, SGD trials fail to converge to the global minimum achieved by our convex program. However, as we increase $K$, the number of trials converging to global minimum gradually increases. Therefore, we show the benign overparameterization impact.

\textbf{Image classification:} We conduct experiments on benchmark image datasets, namely CIFAR-10 \cite{cifar10} and Fashion-MNIST \cite{fashionmnist}. We particularly consider a ten class classification task and use a parallel network with $40$ sub-networks and $100$ hidden neurons, i.e., $(K,m_1,m_2)=(40,100,1)$. In Figure \ref{fig:cifar_mnist}, we plot the test accuracies against wall-clock time, where we include several different optimizers as well as SGD. Moreover, we include a parallel network trained with SGD and Weight Decay (WD) regularization to show the effectiveness of path regularization in \eqref{eq:newmodel_primal1}. We first note that our convex approach achieves both faster convergence and higher final test accuracies for both dataset. However, the performance gain for Fashion-MNIST seems to be significantly less compared to the CIFAR-10 experiment. This is due to the nature of these datasets. More specifically, since CIFAR-10 is a much more challenging dataset, the baseline accuracies are quite low (around $\sim 50\%$) unlike Fashion-MNIST with the baseline accuracies around $\sim 90\%$. Therefore, the accuracy improvement achieved by the convex program seems low in Figure \ref{fig:fmnist}. We also observe that weight decay achieves faster convergence rates however path regularization yields higher final test accuracies. Notice that it is natural to have faster time convergence with weight decay due to its simple form that can be incorporated into gradient-based updates without any computational overhead.

\section{Related Work}

We first note that parallel neural networks training problems were previously investigated by \cite{zhang2019multibranch,haeffele2017global}. Although these studies provided insights into the structure of global minima, they require strict assumptions, e.g., sparsity among sub-networks in Theorem 1 of \cite{haeffele2017global}) and linear activations and hinge loss assumptions in \cite{zhang2019multibranch}, which invalidates extensions to practical settings.

Recently, \cite{pilanci2020neural} studied weight decay regularized two-layer ReLU network training problems and introduced polynomial-time trainable convex formulations. However, their analysis is restricted to standard two-layer ReLU networks, i.e., in the form of $f_{\theta}({\bf X})=\relu{{\bf X}\vec{W}_1} \vec{w}_2$. The reasons for this restriction is that handling more than one ReLU layer is a substantially more challenging optimization problem. As an example, a direct extension of \cite{pilanci2020neural} to three-layer NNs will yield doubly exponential complexity, i.e., $\mathcal{O}(n^{rn^r })$ for a rank-$r$ data matrix, due to the combinatorial behavior of multiple ReLU layers. Thus, they only examined the case with a single ReLU layer (see Table \ref{tab:complexity} for details and the other relevant references in \cite{pilanci2020neural}). In addition, since they only considered standard two-layer ReLU networks, their analysis is not valid for a broader range of NN-based architectures as detailed in Remark \ref{rem:parallel_extensions}. Later on, \cite{ergen2021deep} extended this approach to three-layer ReLU networks. However, since they analyzed $\ell_2$-norm regularized training problem, they had to put unit Frobenius norm constraints on the first layer weights, which does not reflect the settings in practice. In addition, their analysis is restricted to the networks with a single neuron in the second layer (i.e. $m_2=1$) that is in the form of $f_{\theta}({\bf X})=\sum_{k}\relu{\relu{{\bf X}\vec{W}_{1k}} \vec{w}_{2k}}w_{3k}$. Since this architecture only allows a single neuron in the second layer, each sub-network $k$ has an expressive power that is equivalent to a standard two-layer network rather than three-layer. This can also be realized from the definition of the constraint set $\mathcal{C}$ in Theorem \ref{theo:main_theorem}. Specifically, the convex set $\mathcal{C}$ in \cite{ergen2021deep} has decoupled constraints across the hidden layer index $j_1$ whereas our formulation sums the responses over hidden neurons before feeding through the next layer as standard deep networks do. Therefore, this analysis does not reflect the true expressive power of deep networks with $L > 2$. Moreover, as already observed by the authors, the approach in \cite{ergen2021deep} has exponential time complexity when the data matrix is full rank, which is unavoidable.

However, in this paper, we analyze truly deep neural networks in \eqref{eq:parallel_arc} without any assumption on the layer weights. Furthermore, we develop a training algorithm which has fully polynomial-time complexity for all datasets (i.e., even for full rank data) and prove strong global optimality guarantees for this algorithm in Theorem \ref{theo:lowrank_approx}.

\section{Concluding Remarks}
We studied the training problem of path regularized deep parallel ReLU networks, which includes ResNets and standard deep ReLU networks as its special cases. We first showed that the non-convex training problem can be equivalently cast as a single convex optimization problem. Therefore, we achieved the following advantages over the training on the original non-convex formulation: 1) Since our model is convex, it can be globally optimized via standard convex solvers whereas the non-convex formulation trained with optimizers such as SGD might be stuck at a local minimum, 2) Thanks to convexity, our model does not require any sort of heuristics and additional tricks such as learning rate schedule and initialization scheme selection or dropout. More importantly, we proved that the computational complexity required to train our convex model is polynomial-time in terms of the number of data samples $n$ and feature dimension $d$. Thus, we proved the polynomial-time trainability of deep ReLU networks without requiring any impractical assumptions unlike \cite{pilanci2020neural,ergen2021deep}. Notice that we derive an exact convex program only for three-layer networks, however, recently \cite{wang2021parallel} proved that strong duality holds for arbitrarily deep parallel networks. Therefore, a similar analysis can be extended to deeper networks to achieve an equivalent convex program, which is quite promising for future work. In addition, although we only focused on fully connected networks in this paper, our approach can be directly extended to different NN architectures, e.g., convolution networks \cite{ergen2020cnn}, generative adversarial networks \cite{sahiner2021gan}, NNs with batch normalization \cite{ergen2021bn}, and autoregressive models \cite{vikul2021generative}.

redpajama_set_name: RedPajamaArXiv


Each opening morning of a sturgeon spearing season brings with it a special buzz as spearers approach another season with anticipation. Spearers have already selected a location for their shanty and wonder if this will be a year that a sturgeon swims through their hole. Opening day of the 2019 sturgeon spearing season had an added buzz as word spread of a very large fish harvested from Lake Winnebago. DNR staff working at our registration stations heard of the fish and speculated which station the spearer would come to register the fish at and just how big this particular fish would be. Fish records are tracked by weight and the sturgeon spearing record books were almost completely rewritten between 2004-2013 (Figure 1). There had not been a top-11 heaviest fish registered since the 2013 season, would this fish break that streak?
Further, DNR staff had handled fish over the past 10 years that had the potential to break the 200-pound mark, at least when foraging conditions were favorable. Would this fish be one of those tagged fish?
Some of those questions were answered at Wendt's when Jonathan Eiden showed up to register the fish that had created all of the buzz. The fish measured 85.5", which marks one of the longest, if not the longest, lake sturgeon ever harvested from the Winnebago System. The fish weighed an impressive 171.0 pounds, just missing out on the top 11 heaviest fish on record by 0.3 pounds.
The size of this fish was impressive enough, but the lore of this fish was just getting started. A scan for an internal PIT tag detected a PIT tag meaning that this fish had been handled by DNR staff at least once since 1999. 1999 was the year that the DNR started using PIT tags to mark sturgeon and this fish had been marked with PIT tag number 454B440753.
An Excel search by staff working the registration station indicated that this fish also had a sonic transmitter surgically implanted in her abdomen at some point to monitor her movement. At this point, some questions were being answered, but the story was just getting started.
Each registration station calls in their daily harvest numbers to the sturgeon biologist after the 2 PM deadline that spearers must register their fish. While discussing registration numbers with the crew at Wendt's, I queried the PIT tag number of the 85.5" fish and quickly realized that this fish was not only impressive for her size.
She had also been a fish that was the topic of many media publications and a source of discussion during countless DNR presentations and interviews. She was the same fish that had been handled twice by the DNR during spring spawning assessments, most recently below the Shawano Paper Mill Dam in 2012 where she was touted as the 87.5" female lake sturgeon that was estimated to weigh 240 pounds before she had started spawning based off length-weight data from previous harvests).
This fish had been billed as the primary example of the caliber of fish residing in the Winnebago System and a tangible example of the potential record-breaking fish in the system.
We've received several inquiries on what contributed to the disparity in the length and weight estimates of this fish between the 2012 and 2019 recapture events. The difference in length is attributable to a couple of factors. First, our measuring boards go up to 84", so there was a potential error with assigning the length of the last couple of inches.
Further, obtaining accurate measurements from fish in spring and winter can be problematic. In the spring, fish are alive and thrashing around which makes it difficult to estimate length. Conversely, fish harvested during the spear fishery are often frozen to varying degrees which can cause the fish to slightly shrink or not allow staff working stations to properly fold the tail down to obtain the most accurate length estimate.
The difference of 2" is not out of the realm of possibility for a fish of that size. The difference in weight is attributable to abundance of gizzard shad as a forage item. Eight of the eleven heaviest fish on record were harvested between 2010-2013 and all of these fish were shorter than the 171.0 pounder harvested on opening day (Figure 1). Fish harvested during the 2010-2013 seasons were in excellent condition due to strong gizzard shad hatches in 2009, 2010, 2012.
The capture of this fish in 2012 was right in the hay day of fish being in excellent condition, whereas shad hatches in 2017 and 2018 were very weak. This fish was impressive in many facets, but she had the potential to weigh 60-70 more pounds had the last couple of years presented better foraging conditions.
As stated earlier, this fish had been previously captured twice by DNR staff and had a sonic transmitter that allowed her movements to be tracked for a discrete time period. I'd like to take the remainder of this report to present what data the DNR had from this fish.
For starters, the fish was first captured below the Shawano Dam on the Wolf River on April 20, 2004. At that point the fish was 84.0" and marked with a sonic transmitter. Sonic telemetry technology was not as advanced in 2004 and the transmitter had a 1-year battery life (tags currently have up to 10-year battery lives). Female sturgeon typically migrate downstream to the Upriver Lakes or Lake Winnebago shortly after spawning, so a 1-year battery life would allow DNR staff to monitor the downstream movement following spawning.
At that time there were 18 sonic receivers deployed at locations distributed throughout the Winnebago System tributaries. Most of the receivers were deployed throughout the Wolf River from Hwy M just below the Shawano Dam downstream to the Boom Cut near Lake Poygan.
There also were receivers distributed throughout the upper Fox River from Princeton downstream to the Main Street bridge in Oshkosh (the upper Fox River receivers in Princeton, Berlin, and Eureka were not included in Table 1. to conserve space). Each receiver listens for tagged fish and records the tag number (unique fish), date and time of every detection which allows managers to monitor the coarse movement of fish through the Winnebago System.
The fish in question remained in the Hwy CCC area (8 miles downstream of the dam) from April 25, 2004 to May 5, 2004. She was then detected in Stephensville on May 16, 2004 as part of a quick movement downstream that culminated with entry into Lake Poygan on May 17.
The fish then remained in Lake Poygan or Lake Winnebago until June 2, 2004 when she was detected in Winneconne before ultimately entering Lake Winnebago on June 3, 2004. There were a few detections of the fish in the Fox River (Oshkosh) between October 28, 2004 and November 15, 2004.
The next capture of this fish was on April 10, 2012 when she was again captured by DNR tagging crews below the Shawano Paper Mill Dam. This was the last fish handled during a very productive tagging data below the Shawano Dam where 559 fish were handled by DNR staff. The fish measured 87.5" and was roughly ½ spawned out when captured by the crew (photo to right).
harvested during recent spear fisheries.
Displays the number of times the sturgeon was detected on each sonic receiver per date. Following release in 2012, the fish has been at large. Each season, sturgeon spearers have hoped that she would swim through their hole and each spring DNR fisheries staff hope to see her at a spawning site to collect more data. However, neither happened for close to 7 years until she swam through Jonathan Eiden's spearing hole on February 9, 2019. Congratulations Jonathan on the fish of a lifetime (photo below)!
This particular fish has become very popular since being captured in 2012. In fact, there is a wood carving at Sturgeon Park in Shawano that was inspired by this fish! Further, DNR staff frequently reference this fish as an example of the caliber of fish residing in the Winnebago System.
Although this fish is no longer in the population, what she represented still persists. This fish was an example of the big fish that call the Winnebago System home, but I assure you she was not alone. In fact, there have been fish longer than 83" harvested during each of the past three spearing seasons (83.4" fish in 2017, 84.5" fish in 2018, 85.5"fish in 2019). It's only a matter of time until another sturgeon is captured from this population and captivates the attention and dreams of thousands of spearers and sturgeon enthusiasts!


text:
Q: How to select functions that belong in a given extension in PostgreSQL? I am trying to compose a SELECT from pg_catalog that pulls function name, its schema name and the name of extension that created this function.
To pull function list I use (a simplified version)
SELECT pg_proc.proname, pg_namespace.nspname AS proschema
FROM pg_proc
    INNER JOIN pg_namespace ON (pg_namespace.oid = pg_proc.pronamespace)

To select extensions I can run
SELECT pg_extension.extname, pg_namespace.nspname as extschema
FROM pg_catalog.pg_extension
    INNER JOIN pg_catalog.pg_namespace ON (pg_namespace.oid = pg_extension.extnamespace)

Now I am looking for a link between extensions and functions they create when "CREATE EXTENSION". The SELECT should ideally produce a list of extension_name, extension_schema, foo_name, foo_schema

A: All credit for this answer goes to LaurenzAlbe who hinted the solution in the comment. The query that lists functions created by an extension is
SELECT e.extname, ne.nspname AS extschema, p.proname, np.nspname AS proschema
FROM pg_catalog.pg_extension AS e
    INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid)
    INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid)
    INNER JOIN pg_catalog.pg_namespace AS ne ON (ne.oid = e.extnamespace)
    INNER JOIN pg_catalog.pg_namespace AS np ON (np.oid = p.pronamespace)
WHERE d.deptype = 'e'
ORDER BY 1, 3

redpajama_set_name: RedPajamaStackExchange