text:
import sys
from datetime import date, timedelta
from StringIO import StringIO
import psycopg2
import codecs
import zipfile
from settings.const import iff_database_connect

charset = 'cp1252' # yes, this isn't what the documentation suggests

def parse_date(day):
	return date(int(day[4:8]), int(day[2:4]), int(day[0:2]))

def parse_time(time):
	return time[0:2] + ':' + time[2:4] + ':00'

def open_iff(zip,filename, delivery):
	l_content = zip.read(filename+'.dat').decode(charset).split('\r\n')
	if l_content[0] == delivery:
		print "%s.dat matches delivery" % (filename)
	return l_content[1:-1]

def simple_list_writer(conn,filename, arguments, data):
	f = StringIO()
	f.write('\t'.join(arguments) + '\n')
	for y in data:
		f.write('\t'.join([unicode(y[z] or '') for z in arguments]) + '\n')
        f.seek(0)
        cur = conn.cursor()
        if filename == 'country':
            cur.copy_expert("COPY country FROM STDIN USING DELIMITERS '	' CSV HEADER NULL AS '';",f)
        else:
            cur.copy_expert("COPY %s FROM STDIN USING DELIMITERS '	' CSV HEADER" % (filename),f)
        cur.close()
	f.close()

def simple_dict_writer(conn,filename, arguments, data):
	f = StringIO()
	f.write('\t'.join(arguments) + '\n')
	for x, y in data.items():
		f.write('\t'.join([unicode(x)] + [unicode(y[z] or '') for z in arguments[1:]]) + '\n')
        f.seek(0)
        cur = conn.cursor()
        cur.copy_expert("COPY %s FROM STDIN USING DELIMITERS '	' CSV HEADER" % (filename),f)
        cur.close()
	f.close()

def simple_dict_list_writer(conn,filename, arguments, data):
	f = StringIO()
	f.write('\t'.join(arguments) + '\n')
	for x, y in data.items():
		for u in y:
			f.write('\t'.join([unicode(x)] + [unicode(u[z] or '') for z in arguments[1:]]) + '\n')
        f.seek(0)
        cur = conn.cursor()
        cur.copy_expert("COPY %s FROM STDIN USING DELIMITERS '	' CSV HEADER" % (filename),f)
        cur.close()
	f.close()

def parse_timetables(zip,delivery):
	l_timetables = open_iff(zip,'timetbls', delivery)

	timetables = {}
	current_id = None
	current_record = {}
	s_stationshort = None
	s_index = 0

	for x in l_timetables:
		if x[0] == '#':
			if current_id is not None:
				timetables[current_id] = current_record
			s_index = 0
			current_id = int(x[1:])
			current_record = {'service': [], 'validity': [], 'transport': [], 'attribute': [], 'stop': [], 'platform': []}
		elif x[0] == '%':
			s_companynumber, s_servicenumber, s_variant, s_firststop, s_laststop, s_servicename = x[1:].split(',')
			current_record['service'].append({'company': int(s_companynumber), 'service': int(s_servicenumber), 'variant': s_variant.strip(), 'first': int(s_firststop), 'last': int(s_laststop), 'name': s_servicename.strip()})
		elif x[0] == '-':
			v_footnote, v_firststop, v_laststop = x[1:].split(',')
			current_record['validity'].append({'footnote': v_footnote, 'first': int(v_firststop), 'last': int(v_laststop)})
		elif x[0] == '&':
			t_mode, t_firststop, t_laststop = x[1:].split(',')
			current_record['transport'].append({'mode': t_mode.strip(), 'first': int(t_firststop), 'last': int(t_laststop)})
		elif x[0] == '*':
			t_code, t_firststop, t_laststop, t_unknown = x[1:].split(',')
			current_record['attribute'].append({'code': t_code.strip(), 'first': int(t_firststop), 'last': int(t_laststop), 'unknown': int(t_unknown)})
		elif x[0] == '>':
			s_index += 1
			s_stationshort, s_departuretime = x[1:].split(',')
			s_stationshort = s_stationshort.strip()
			current_record['stop'].append({'station': s_stationshort, 'index': s_index, 'arrivaltime': None, 'departuretime': parse_time(s_departuretime)})
		elif x[0] == '.':
			s_index += 1
			s_stationshort, s_arrivaldeparturetime = x[1:].split(',')
			s_stationshort = s_stationshort.strip()
			both = parse_time(s_arrivaldeparturetime)
			current_record['stop'].append({'station': s_stationshort, 'index': s_index, 'arrivaltime': both, 'departuretime': both})
		elif x[0] == ';':
			s_index += 1
			s_stationshort = x[1:].split(',')
			s_stationshort = s_stationshort.strip()
			current_record['stop'].append({'station': s_stationshort, 'index': s_index, 'arrivaltime': None, 'departuretime': None})
		elif x[0] == '+':
			s_index += 1
			s_stationshort, s_arrivaltime, s_departuretime = x[1:].split(',')
			s_stationshort = s_stationshort.strip()
			current_record['stop'].append({'station': s_stationshort, 'index': s_index, 'arrivaltime': parse_time(s_arrivaltime), 'departuretime': parse_time(s_departuretime)})
		elif x[0] == '?':
			s_arrivalplatform, s_departureplatform, footnote = x[1:].split(',')
			current_record['platform'].append({'index': s_index,'station': s_stationshort,'arrival': s_arrivalplatform.strip(), 'departure': s_departureplatform.strip(), 'footnote': int(footnote)})
			if s_arrivalplatform[0] <> s_departureplatform[0]:
				print current_id, s_stationshort, x
		elif x[0] == '<':
			s_index += 1
			s_stationshort, s_arrivaltime = x[1:].split(',')
			s_stationshort = s_stationshort.strip()
			current_record['stop'].append({'station': s_stationshort, 'index': s_index, 'arrivaltime': parse_time(s_arrivaltime), 'departuretime': None})

	if current_id is not None:
		timetables[current_id] = current_record

	return timetables

def sql_timetables(conn,data):
	f = {}
	a = {}
	f['service'] = StringIO()
	f['validity'] = StringIO()
	f['transport'] = StringIO()
	f['attribute'] = StringIO()
	f['stop'] = StringIO()
	f['platform'] = StringIO()

	a['service'] = ['company', 'service', 'variant', 'first', 'last', 'name']
	a['validity'] = ['footnote', 'first', 'last']
	a['transport'] = ['mode', 'first', 'last']
	a['attribute'] = ['code', 'first', 'last']
	a['stop'] = ['index', 'station', 'arrivaltime', 'departuretime']
	a['platform'] = ['index','station','arrival', 'departure', 'footnote']

	for x in f.keys():
		f[x].write('\t'.join(['serviceid'] + a[x]) + '\n')

	for x, y in data.items():
		for z in f.keys():
			for u in y[z]:
				f[z].write('\t'.join([unicode(x)] + [unicode(u[w] or '') for w in a[z]]) + '\n')
        cur = conn.cursor()
	for filename,f in f.items():
            f.seek(0)
            cur.copy_expert("COPY timetable_%s FROM STDIN USING DELIMITERS '	' CSV HEADER" % (filename),f)
            f.close()
        cur.close()

def parse_timezones(zip,delivery):
	l_timezones = open_iff(zip,'timezone', delivery)

	timezones = {}
	current_id = None
	current_values = []

	for x in l_timezones:
		if x[0] == '#':
			if current_id is not None:
				timezones[current_id] = current_values
			current_id = int(x[1:])
			current_values = []
		else:
			difference, firstday, lastday = x[1:].split(',')
			if x[0] == '-':
				difference = int(difference) * -1
			elif x[0] == '+':
				difference = int(difference)

			current_values.append({'difference': difference, 'firstday': parse_date(firstday), 'lastday': parse_date(lastday)})

	if current_id is not None:
		timezones[current_id] = current_values

	return timezones

def sql_timezones(conn,data):
	simple_dict_list_writer(conn,'timezone', ['tznumber', 'difference', 'firstday', 'lastday'], data)

def parse_transattributequestions(zip,delivery):
	l_aq = open_iff(zip,'trnsaqst', delivery)

	aq = {}
	current_id = None
	current_record = {}

	for x in l_aq:
		if x[0] == '#':
			if current_id is not None:
				aq[current_id] = current_record
			q_code, q_type, q_question = x[1:].split(',')
			current_id = q_code.strip()
			current_record = {'attributes': [], 'inclusive': bool(q_type == '1'), 'question': q_question.strip()}
		elif x[0] == '-':
			current_record['attributes'].append(x[1:].strip())

	if current_id is not None:
		aq[current_id] = current_record

	return aq

def sql_transattributequestions(conn,data):
	f = StringIO()
	f.write('\t'.join(['code', 'inclusive', 'question', 'transattr']) + '\n')
	for x, y in data.items():
		for z in y['attributes']:
			f.write('\t'.join([unicode(u or '') for u in [x, y['inclusive'], y['question'], z]]) + '\n')
        f.seek(0)
        cur = conn.cursor()
        cur.copy_expert("COPY trnsaqst FROM STDIN USING DELIMITERS '	' CSV HEADER NULL AS '';",f)
        cur.close()
	f.close()

def parse_footnotes(zip,delivery):
	l_footnotes = open_iff(zip,'footnote', delivery)

	footnotes = {}
	current_id = None

	for x in l_footnotes:
		if x[0] == '#':
			current_id = int(x[1:])
		else:
			footnotes[current_id] = [y == '1' for y in x]

	return footnotes

def sql_footnotes(conn,delivery, data):
	f = StringIO()
	f.write('\t'.join(['footnote', 'servicedate']) + '\n')
	for x, y in data.items():
		for z in range(0, len(y)):
			if y[z] == True:
				f.write('\t'.join([unicode(x), unicode(delivery['firstday'] + timedelta(days=z))]) + '\n')
        f.seek(0)
        cur = conn.cursor()
        cur.copy_expert("COPY footnote FROM STDIN USING DELIMITERS '	' CSV HEADER NULL AS '';",f)
        cur.close()
	f.close()

def parse_changes(zip,delivery):
	l_changes = open_iff(zip,'changes', delivery)

	changes = {}
	current_id = None
	current_records = None

	for x in l_changes:
		if x[0] == '#':
			current_id = x[1:].strip()
                        changes[current_id] = []
		else:
			c_from, c_to, c_change = [y for y in x[1:].split(',')]
			changes[current_id].append({'from': int(c_from), 'to': int(c_to), 'change': c_change})

	return changes

def sql_changes(conn,data):
	simple_dict_list_writer(conn,'changes', ['station', 'from', 'to', 'change'], data)

def parse_countries(zip,delivery):
	l_countries = open_iff(zip,'country', delivery)

	countries = {}
	for country in l_countries:
		c_code, c_inland, c_name = country.split(',')
		countries[c_code.strip()] = {'inland': bool(c_inland), 'name': c_name.strip()}

	return countries

def sql_countries(conn,countries):
	simple_dict_writer(conn,'country', ['code', 'inland', 'name'], countries)

def parse_transattributes(zip,delivery):
	l_transattributes = open_iff(zip,'trnsattr', delivery)

	transattributes = {}
	for attribute in l_transattributes:
		a_code, a_pcode, a_description = attribute.split(',')
		transattributes[a_code.strip()] = {'processingcode': int(a_pcode), 'description': a_description.strip()}

	return transattributes

def sql_transattributes(conn,data):
	simple_dict_writer(conn,'trnsattr', ['code', 'processingcode', 'description'], data)

def parse_companies(zip,delivery):
	l_companies = open_iff(zip,'company', delivery)

	companies = {}
	for company in l_companies:
		c_number, c_code, c_name, c_time = company.split(',')
		companies[int(c_number)] = {'code': c_code.strip(), 'name': c_name.strip(), 'timeturn': parse_time(c_time)}

	return companies

def sql_companies(conn,data):
	simple_dict_writer(conn,'company', ['number', 'code', 'name', 'timeturn'], data)

def parse_connectionmodes(zip,delivery):
	l_cms = open_iff(zip,'connmode', delivery)

	cms = {}
	for cm in l_cms:
		c_code, c_type, c_description = cm.split(',')
		cms[c_code.strip()] = {'type': int(c_type), 'description': c_description.strip()}

	return cms

def sql_connectionmodes(conn,data):
	simple_dict_writer(conn,'connmode', ['code', 'type', 'description'], data)

def parse_continuousconnections(zip,delivery):
	l_ccs = open_iff(zip,'contconn', delivery)

	ccs = []
	for cc in l_ccs:
		c_from, c_to, c_time, c_mode = cc.split(',')
		ccs.append({'from': c_from.strip(), 'to': c_to.strip(), 'time': int(c_time) * 60, 'mode': c_mode.strip()})

	return ccs

def sql_continuousconnections(conn,data):
	simple_list_writer(conn,'contconn', ['from', 'to', 'time', 'mode'], data)

def parse_transmodes(zip,delivery):
	l_transmode = open_iff(zip,'trnsmode', delivery)

	transmodes = {}
	for transmode in l_transmode:
		t_code, t_description = transmode.split(',')
		transmodes[t_code.strip()] = {'description': t_description.strip()}

	return transmodes

def sql_transmodes(conn,data):
	simple_dict_writer(conn,'trnsmode', ['code', 'description'], data)

def parse_stations(zip,delivery):
	l_stations = open_iff(zip,'stations', delivery)

	stations = {}
	for station in l_stations:
		changes, shortname, mintime, _maxtime, country, timezone, attribute, x, y, name = station.split(',')
		stations[shortname.strip()] = {'changes': changes, 'layovertime': int(mintime) * 60, 'country': country.strip(), 'timezone': int(timezone), 'x': int(x) * 10, 'y': int(y) * 10, 'name': name.strip()}

		# Multiply the RD coordinate by 10
		# Multiply the minimum transfer time by 60s

	return stations

def sql_stations(conn,data):
	simple_dict_writer(conn,'station', ['shortname', 'changes', 'layovertime', 'country', 'timezone', 'x', 'y', 'name'], data)

def create_schema(conn):
    cur = conn.cursor()
    cur.execute("""
create temporary table country(code varchar(4) primary key, inland boolean not null, name varchar(29) not null);
create temporary table company(company integer primary key, code varchar(9) not null, name varchar(29) not null, timeturn time);
create temporary table delivery(company integer references company, firstday date, lastday date, versionnumber integer, description varchar(29));
create temporary table timezone(tznumber integer primary key, difference integer, firstday date not null, lastday date not null);
create temporary table station(shortname varchar(6) primary key, trainchanges smallint, layovertime integer, country varchar(4) references country, timezone integer references timezone, x integer, y integer, name varchar(29));
create temporary table trnsattr (code varchar(4) primary key, processingcode smallint not null, description varchar(30));
create temporary table trnsaqst(code varchar(3), inclusive boolean, question varchar(29) not null, transattr varchar(4) references trnsattr, primary key (code, transattr));
-- documentation states 'mode' instead of attr
create temporary table trnsmode(code varchar(4) primary key, description varchar(29));
-- create table trnsmqst(code varchar(3), question varchar(29) not null, transmode varchar(4) references trnsmode, primary key(code, transmode));
create temporary table connmode(code varchar(4) primary key, connectiontype smallint not null, description varchar(29));
create temporary table contconn(fromstation varchar(6) references station, tostation varchar(6) references station, connectiontime integer not null, connectionmode varchar(4) references connmode not null, primary key(fromstation, tostation, connectionmode));
create temporary table footnote(footnote integer NOT NULL, servicedate date);
create temporary table timetable_service (serviceid integer not null, companynumber integer references company, servicenumber integer, variant integer, firststop numeric(3,0), laststop numeric(3,0), servicename varchar(29));
create temporary table timetable_validity (serviceid integer not null, footnote integer NOT NULL, firststop numeric(3,0), laststop numeric(3,0));
create temporary table timetable_transport (serviceid integer not null, transmode varchar(4) references trnsmode, firststop numeric(3,0), laststop numeric(3,0));
create temporary table timetable_attribute (serviceid integer not null, code varchar(4) references trnsattr, firststop numeric(3,0), laststop numeric(3,0));
create temporary table timetable_stop (serviceid integer, idx integer, station varchar(6) references station, arrivaltime char(8), departuretime char(8), primary key(serviceid, idx));
create temporary table timetable_platform (serviceid integer, idx integer, station varchar(6) references station, arrival varchar(4), departure varchar(4), footnote integer, primary key(serviceid, idx), foreign key (serviceid, idx) references timetable_stop);
create temporary table changes(station varchar(6) references station, fromservice integer not null, toservice integer not null, possiblechange smallint);
CREATE OR REPLACE FUNCTION
toseconds(time24 text, shift24 integer) RETURNS integer AS $$
SELECT total AS time
FROM
(SELECT
  (cast(split_part($1, ':', 1) as int4) * 3600)      -- hours
+ (cast(split_part($1, ':', 2) as int4) * 60)        -- minutes
+ CASE WHEN $1 similar to '%:%:%' THEN (cast(split_part($1, ':', 3) as int4)) ELSE 0 END -- seconds when applicable
+ (shift24 * 86400) as total --Add 24 hours (in seconds) when shift occured
) as xtotal
$$ LANGUAGE SQL;

CREATE OR REPLACE FUNCTION
to32time(secondssincemidnight integer) RETURNS text AS $$
SELECT lpad(floor((secondssincemidnight / 3600))::text, 2, '0')||':'||lpad(((secondssincemidnight % 3600) / 60)::text, 2,
'0')||':'||lpad((secondssincemidnight % 60)::text, 2, '0') AS time
$$ LANGUAGE SQL;

CREATE OR REPLACE FUNCTION
route(servicenumber integer,variant integer) RETURNS integer AS $$
SELECT CASE WHEN (servicenumber = 0 and variant is null) THEN NULL
            WHEN (servicenumber = 0 and variant between 900000 and 999999) THEN NULL
            WHEN (coalesce(nullif(servicenumber,0),variant) between 28000 and 28999) THEN NULL
            WHEN (coalesce(nullif(servicenumber,0),variant) between 0 and 99)    THEN (coalesce(nullif(servicenumber,0),variant)/10)*10
            WHEN (coalesce(nullif(servicenumber,0),variant) between 100 and 109) THEN 100
            WHEN (coalesce(nullif(servicenumber,0),variant) between 140 and 149) THEN 140
            WHEN (coalesce(nullif(servicenumber,0),variant) between 240 and 249) THEN 240
            WHEN (coalesce(nullif(servicenumber,0),variant) between 430 and 439) THEN 430
            WHEN (coalesce(nullif(servicenumber,0),variant) between 440 and 449) THEN 440
            WHEN (coalesce(nullif(servicenumber,0),variant) between 100 and 99999) THEN (coalesce(nullif(servicenumber,0),variant)/100)*100
            WHEN (coalesce(nullif(servicenumber,0),variant) between 690000 and 699999) THEN ((coalesce(nullif(servicenumber,0),variant)-690000)/100)*100
            WHEN (coalesce(nullif(servicenumber,0),variant) between 700000 and 709999) THEN ((coalesce(nullif(servicenumber,0),variant)-700000)/100)*100
            WHEN (coalesce(nullif(servicenumber,0),variant) between 710000 and 719999) THEN ((coalesce(nullif(servicenumber,0),variant)-710000)/100)*100
            WHEN (coalesce(nullif(servicenumber,0),variant) between 720000 and 729999) THEN ((coalesce(nullif(servicenumber,0),variant)-720000)/100)*100
            WHEN (coalesce(nullif(servicenumber,0),variant) between 730000 and 739999) THEN ((coalesce(nullif(servicenumber,0),variant)-730000)/100)*100
            WHEN (coalesce(nullif(servicenumber,0),variant) between 740000 and 749999) THEN ((coalesce(nullif(servicenumber,0),variant)-740000)/100)*100
            WHEN (coalesce(nullif(servicenumber,0),variant) between 750000 and 759999) THEN ((coalesce(nullif(servicenumber,0),variant)-750000)/100)*100
            WHEN (coalesce(nullif(servicenumber,0),variant) between 800000 and 809999) THEN ((coalesce(nullif(servicenumber,0),variant)-800000)/100)*100
            WHEN (coalesce(nullif(servicenumber,0),variant) between 860000 and 869999) THEN ((coalesce(nullif(servicenumber,0),variant)-860000)/100)*100
            WHEN (coalesce(nullif(servicenumber,0),variant) between 900000 and 999999) THEN ((coalesce(nullif(servicenumber,0),variant)-900000)/100)*100
            ELSE null END as trainnumber
$$ LANGUAGE SQL;

CREATE OR REPLACE FUNCTION
line_id(companynumber integer, transmode varchar,servicenumber integer,variant integer, stops varchar[]) RETURNS varchar AS $$
SELECT CASE WHEN (transmode in ('NSB','NSS')) THEN concat_ws(':',companynumber,transmode,least(stops[array_lower(stops,1)],stops[array_upper(stops,1)]),greatest(stops[array_lower(stops,1)],stops[array_upper(stops,1)]))
            WHEN (route(servicenumber,variant) IS NOT NULL) THEN concat_ws(':',companynumber,transmode,route(servicenumber,variant))
            ELSE concat_ws(':',companynumber,transmode,least(stops[array_lower(stops,1)],stops[array_upper(stops,1)]),greatest(stops[array_lower(stops,1)],stops[array_upper(stops,1)]))
            END as line_id
$$ LANGUAGE SQL;""")

def configDB(conn):
    cur = conn.cursor()
    cur.execute("""

CREATE OR REPLACE FUNCTION add32time(departuretime text, seconds integer) RETURNS text AS $$ SELECT lpad(floor((total / 3600))::text, 2,
'0')||':'||lpad(((total % 3600) / 60)::text, 2, '0')||':'||lpad((total % 60)::text, 2, '0') AS arrival_time FROM (SELECT (cast(split_part($1, ':', 1)
as int4) * 60 + cast(split_part($1, ':', 2) as int4)) * 60 + cast(split_part($1, ':', 3) as int4) + coalesce($2, 0) as total) as xtotal $$ LANGUAGE
SQL;

DELETE FROM timetable_validity WHERE footnote not in (
SELECT footnote FROM (select footnote,max(servicedate) as enddate from footnote group by footnote) as x WHERE enddate >= current_date - interval '1
days');
DELETE FROM timetable_service WHERE serviceid not in (SELECT serviceid FROM timetable_validity);

update timetable_stop set departuretime = add32time(departuretime,3600),arrivaltime = add32time(arrivaltime,3600) where station in (select shortname
from station where country = 'GB');

update trnsmode set description = 'Stoptrein' where description = 'stoptrein';
INSERT INTO company VALUES ('999','EUROSTAR','Eurostar','00:00:00');
UPDATE timetable_service
SET companynumber = 980
WHERE
companynumber = 960 AND
serviceid in (select serviceid from timetable_transport where transmode = 'TGV');
UPDATE timetable_service
SET companynumber = 300
WHERE
companynumber = 960 AND
serviceid in (select serviceid from timetable_transport where transmode = 'THA');
UPDATE timetable_service
SET companynumber = 999
WHERE
companynumber = 960 AND
serviceid in (select serviceid from timetable_transport where transmode = 'ES');

CREATE VIEW timetable as (
SELECT
line_id(companynumber,transmode,servicenumber,variant,array_agg(station) over (PARTITION BY serviceid,transmode,coalesce(servicenumber,variant) ORDER
BY idx range between unbounded preceding and unbounded following)) as line_id,
companynumber,serviceid,footnote,transmode,servicenumber,variant,servicename,idx,row_number() over(PARTITION BY
serviceid,transmode,coalesce(servicenumber,variant) ORDER BY idx ASC) as
stoporder,arrivaltime,CASE WHEN (stoptype = 'LAST') THEN arrivaltime ELSE departuretime END as departuretime,station,arrivalplatform,departureplatform,
md5(string_agg(station||':'||coalesce(departureplatform,'0')||':'||coalesce(arrivalplatform,'0')||':'||attrs::text,'>') over (PARTITION BY
serviceid,coalesce(servicenumber,variant))) as patterncode,attrs,
((not (ARRAY['NUIT']::varchar[] <@ attrs)) and stoptype <> 'FIRST') as foralighting,
((not (ARRAY['NIIN']::varchar[] <@ attrs)) and stoptype <> 'LAST') as forboarding
FROM (
SELECT companynumber,serviceid,v.footnote,servicenumber,variant,servicename,transmode,idx,departuretime as
arrivaltime,departuretime,station,departure as arrivalplatform,departure as departureplatform,attrs,'FIRST' as stoptype
FROM timetable_service as s LEFT JOIN timetable_stop USING (serviceid) LEFT JOIN timetable_platform USING (serviceid,station,idx) LEFT JOIN
timetable_validity as v USING (serviceid)
     LEFT JOIN (select serviceid,transmode,firststop,laststop,generate_series(firststop::int,laststop::int) as idx from timetable_transport) as
timetable_transport USING (serviceid,idx)
     LEFT JOIN (select serviceid,array_agg(code) as attrs,generate_series(firststop::int,laststop::int) as idx from timetable_attribute GROUP BY
serviceid,idx) as timetable_attribute USING (serviceid,idx)
WHERE idx = s.firststop AND idx != timetable_transport.laststop
UNION
SELECT
companynumber,serviceid,v.footnote,servicenumber,variant,servicename,transmode,idx,coalesce(arrivaltime,'00:00:00'),departuretime,station,arrival as
arrivalplatform,departure  as departureplatform,attrs,
CASE WHEN (timetable_transport.firststop < idx and timetable_transport.laststop  = idx) THEN 'LAST'
     WHEN (timetable_transport.laststop > idx and timetable_transport.firststop = idx) THEN 'FIRST'
     ELSE 'INTERMEDIATE' END as stoptype
FROM timetable_service as s LEFT JOIN timetable_stop USING (serviceid) LEFT JOIN timetable_platform USING (serviceid,station,idx) LEFT JOIN
timetable_validity as v USING (serviceid)
     LEFT JOIN (select serviceid,transmode,firststop,laststop,generate_series(firststop::int,laststop::int) as idx from timetable_transport) as timetable_transport USING (serviceid,idx)
     LEFT JOIN (select serviceid,array_agg(code) as attrs,generate_series(firststop::int,laststop::int) as idx from timetable_attribute GROUP BY serviceid,idx) as timetable_attribute USING (serviceid,idx)
WHERE idx between s.firststop+1 and s.laststop-1
UNION
SELECT companynumber,serviceid,v.footnote,servicenumber,variant,servicename,transmode,idx,arrivaltime,arrivaltime as departuretime,station,arrival as
arrivalplatform,arrival as departureplatform,attrs, 'LAST' as stoptype
FROM timetable_service as s LEFT JOIN timetable_stop USING (serviceid) LEFT JOIN timetable_platform USING (serviceid,station,idx) LEFT JOIN timetable_validity as v USING (serviceid)
     LEFT JOIN (select serviceid,transmode,firststop,laststop,generate_series(firststop::int,laststop::int) as idx from timetable_transport) as timetable_transport USING (serviceid,idx)
     LEFT JOIN (select serviceid,array_agg(code) as attrs,laststop as idx from timetable_attribute GROUP BY serviceid,idx) as timetable_attribute USING (serviceid,idx)
WHERE idx = s.laststop AND idx != timetable_transport.firststop
) as x
order by serviceid,servicenumber,footnote,variant,stoporder);

create temporary table passtimes as (
SELECT line_id,companynumber,serviceid,footnote,transmode,servicenumber,variant,servicename,idx,station,platform,
row_number() over(PARTITION BY line_id,serviceid,transmode,coalesce(servicenumber,variant) ORDER BY stoporder,arrivaltime ASC) as stoporder
,arrivaltime,departuretime,md5(string_agg(station||':'||coalesce(platform,'0'),'>') OVER (PARTITION BY
line_id,serviceid,transmode,coalesce(servicenumber,variant))) as patterncode,attrs,foralighting,forboarding
FROM
(SELECT
line_id,companynumber,serviceid,footnote,transmode,servicenumber,variant,servicename,idx,stoporder,station,departureplatform as platform,
arrivaltime,departuretime,patterncode,CASE WHEN ('{NULL}' = attrs) THEN NULL ELSE attrs END as attrs,foralighting,forboarding
FROM
timetable WHERE arrivalplatform = departureplatform or arrivalplatform is null
UNION
SELECT
line_id,companynumber,serviceid,footnote,transmode,servicenumber,variant,servicename,idx,stoporder,station,arrivalplatform as platform,
arrivaltime,arrivaltime as departuretime,patterncode,CASE WHEN ('{NULL}' = attrs) THEN NULL ELSE attrs END as attrs,foralighting,false as forboarding
FROM
timetable WHERE arrivalplatform <> departureplatform
UNION
SELECT
line_id,companynumber,serviceid,footnote,transmode,servicenumber,variant,servicename,idx,stoporder,station,departureplatform as platform,
to32time(toseconds(arrivaltime,0)+15) as arrivaltime,departuretime,patterncode,CASE WHEN ('{NULL}' = attrs) THEN NULL ELSE attrs END as attrs,false as foralighting,forboarding
FROM
timetable WHERE arrivalplatform <> departureplatform) as x
ORDER BY line_id,servicenumber,variant,serviceid,stoporder
);
""")

def sql_delivery(conn,delivery):
	f = StringIO()
	f.write('\t'.join(['companynumber', 'firstday', 'lastday', 'versionnumber', 'description']) + '\n')
	f.write('\t'.join([unicode(delivery[x] or '') for x in ['companynumber', 'firstday', 'lastday', 'versionnumber', 'description']]) + '\n')
        f.seek(0)
        cur = conn.cursor()
        cur.copy_expert("COPY delivery FROM STDIN USING DELIMITERS '	' CSV HEADER NULL AS ''",f)
        cur.close()
	f.close()
	f.close()

def load(path,filename):
    zip = zipfile.ZipFile(path+'/'+filename,'r')
    delivery = zip.read('delivery.dat').decode(charset).split('\r\n')[0]
    number, firstday, lastday, versionnumber, description = delivery[1:].split(',')

    delivery = {'companynumber': int(number), 'firstday': parse_date(firstday), 'lastday': parse_date(lastday), 'versionnumber': int(versionnumber), 'description': description.strip()}

    countries = parse_countries(zip,delivery)
    companies = parse_companies(zip,delivery)
    transmodes = parse_transmodes(zip,delivery)
    transattributes = parse_transattributes(zip,delivery)
    timezones = parse_timezones(zip,delivery)
    transattributequestions = parse_transattributequestions(zip,delivery)
    footnotes = parse_footnotes(zip,delivery)
    continuousconnections = parse_continuousconnections(zip,delivery)
    connectionmodes = parse_connectionmodes(zip,delivery)
    changes = parse_changes(zip,delivery)
    timetables = parse_timetables(zip,delivery)
    stations = parse_stations(zip,delivery)

    print "Received file from %s for period %s to %s (%s)" % (companies[delivery['companynumber']]['name'], delivery['firstday'], delivery['lastday'], delivery['description'])

    conn = psycopg2.connect(iff_database_connect)
    create_schema(conn)
    sql_countries(conn,countries)
    sql_timezones(conn,timezones)
    sql_footnotes(conn,delivery, footnotes)
    sql_stations(conn,stations)
    sql_companies(conn,companies)
    sql_delivery(conn,delivery)
    sql_transattributes(conn,transattributes)
    sql_transattributequestions(conn,transattributequestions)
    sql_transmodes(conn,transmodes)
    sql_connectionmodes(conn,connectionmodes)
    sql_continuousconnections(conn,continuousconnections)
    sql_changes(conn,changes)
    sql_timetables(conn,timetables)
    configDB(conn)
    return (delivery,conn)

if __name__ == '__main__':
    load(sys.argv[1],sys.argv[2])


text:
window.jsCommon = window.jsCommon || {};
window.powerbi = window.powerbi || {};
window.powerbitests = window.powerbitests || {};
window.debug = window.debug || {};
window.InJs = window.InJs || {};
window.Microsoft = window.Microsoft || {};

// Extra libraries used in tests
require("script!../../../node_modules/jasmine-jquery/lib/jasmine-jquery.js");
require("script!../../../src/Clients/Externals/ThirdPartyIP/QuillJS/quill.min.js");
require("script!../../../src/Clients/Externals/ThirdPartyIP/noUiSlider/nouislider.min.js");

require("./typedefs/typedefs.ts");

require("./common.ts");
require("./mocks.ts");
require("./helpers/helpers.ts");
require("./helpers/forecastTestHelpers.ts");
require("./helpers/performanceTestsHelpers.ts");
require("./helpers/slicerHelper.ts");
require("./helpers/tableDataViewHelper.ts");
require("./helpers/tablixHelper.ts");
require("./helpers/kpiHelper.ts");
require("./helpers/referenceLineHelper.ts");
require("./helpers/trendLineHelper.ts");
require("./jasmineMatchers/toEqualDeep.ts");
require("./jasmineMatchers/toEqualSQExpr.ts");

require("./sqFieldDef.ts");

require("./customVisuals/helpers/dataViewHelpers.ts");
require("./customVisuals/helpers/visualTestHelpers.ts");
require("./customVisuals/VisualBuilderBase.ts");
require("./extensibility/extensibilityMocks.ts");
require("./extensibility/extensibilityHelpers.ts");
require("./customVisuals/sampleDataViews/DataViewBuilder.ts");
require("./customVisuals/sampleDataViews/SalesByCountryData.ts");
require("./customVisuals/sampleDataViews/SankeyDiagramData.ts");
require("./customVisuals/sampleDataViews/MekkoChartData.ts");
require("./customVisuals/sampleDataViews/BulletChartData.ts");
require("./customVisuals/sampleDataViews/CarLogosData.ts");
require("./customVisuals/sampleDataViews/CountriesData.ts");
require("./customVisuals/sampleDataViews/ChordChartData.ts");
require("./customVisuals/sampleDataViews/SalesByDayOfWeekData.ts");
require("./customVisuals/sampleDataViews/EnhancedScatterChartData.ts");
require("./customVisuals/sampleDataViews/forceGraphData.ts");
require("./customVisuals/sampleDataViews/GanttData.ts");
require("./customVisuals/sampleDataViews/AsterPlotData.ts");
require("./customVisuals/sampleDataViews/TimelineData.ts");
require("./customVisuals/sampleDataViews/valueByAgeData.ts");
require("./customVisuals/sampleDataViews/ProductSalesByDateData.ts");
require("./customVisuals/sampleDataViews/PulseChartData.ts");
require("./customVisuals/sampleDataViews/LineDotChartData.ts");
require("./customVisuals/sampleDataViews/ValueByNameData.ts");
require("./customVisuals/sampleDataViews/ValueByNameGroupData.ts");

redpajama_set_name: RedPajamaGithub


text:
<?php
/**
 * extends the Decimal Field
 *
 * @package framework
 * @subpackage model
 */
class ExtendedDatetime extends DataExtension {

    protected $locale_formats = array(
        'en_US' => 'm/d/Y g:i A', //
        'de_DE' => 'd.m.Y H:i \U\h\r',
        'en_GB' => 'd/m/Y H:i'
    );

    protected function getFormat(){
        if(array_key_exists(i18n::get_locale(), $this->locale_formats)) $locale = i18n::get_locale();
        else $locale = 'en_US';

        return $this->locale_formats[$locale];
    }

    /**
     * returns the value formatet in the current locales format
     *
     * @return string
     */
    public function NiceLocale(){

        if($this->owner->value){
            return date($this->getFormat(), strtotime($this->owner->value));
        }
    }
}


text:
using System;
using System.Reflection;
using System.Runtime.InteropServices;

// Allgemeine Informationen über eine Assembly werden über die folgenden
// Attribute gesteuert. Ändern Sie diese Attributwerte, um die Informationen zu ändern,
// die mit einer Assembly verknüpft sind.
[assembly: AssemblyTitle("DemoProject")]
[assembly: AssemblyDescription("DemoProject")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("DemoProject")]
[assembly: AssemblyCopyright("")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]


text:
<?xml version="1.0" ?>
<!--
  Licensed to the Apache Software Foundation (ASF) under one or more
  contributor license agreements.  See the NOTICE file distributed with
  this work for additional information regarding copyright ownership.
  The ASF licenses this file to You under the Apache License, Version 2.0
  (the "License"); you may not use this file except in compliance with
  the License.  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
  -->

<schema name="test" version="1.4">

  <fieldType name="tint" class="${solr.tests.IntegerFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8"/>
  <fieldType name="tfloat" class="${solr.tests.FloatFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8"/>
  <fieldType name="tlong" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8"/>

  <!-- tdouble class can't be randomized, because BBoxField requires TrieDoubleField.  See SOLR-11111. -->
  <fieldType name="tdouble" class="solr.TrieDoubleField" docValues="${solr.tests.numeric.dv}" precisionStep="8"/>

  <fieldType name="tdoubleDV" class="${solr.tests.DoubleFieldType}" precisionStep="8" docValues="true"/>

  <fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/>

  <fieldType name="boolean" class="solr.BoolField"/>

  <fieldType name="string" class="solr.StrField" sortMissingLast="true"/>

  <fieldType name="point2D" class="solr.PointType" dimension="2" subFieldType="tdouble"/>

  <fieldType name="srpt_geohash" class="solr.SpatialRecursivePrefixTreeFieldType"
             prefixTree="geohash" distanceUnits="degrees"
  />
  <fieldType name="srpt_quad" class="solr.SpatialRecursivePrefixTreeFieldType"
             prefixTree="quad" distanceUnits="degrees"
  />
  <fieldType name="srpt_packedquad" class="solr.SpatialRecursivePrefixTreeFieldType"
             prefixTree="packedQuad" distanceUnits="degrees"
  />
  <fieldType name="srpt_100km" class="solr.SpatialRecursivePrefixTreeFieldType"
             maxDistErr="100" distanceUnits="kilometers"
  />

  <fieldType name="srptgeom" class="solr.RptWithGeometrySpatialField"/>

  <fieldType name="srptgeom_geo3d" class="solr.RptWithGeometrySpatialField"
             spatialContextFactory="Geo3D" prefixTree="geohash" planetModel="wgs84"/><!-- or sphere -->

  <fieldType name="srptgeom_s2_geo3d" class="solr.RptWithGeometrySpatialField"
             spatialContextFactory="Geo3D" prefixTree="s2" planetModel="sphere"/>

  <fieldType name="bbox" class="solr.BBoxField"
             numberType="tdoubleDV" distanceUnits="degrees" storeSubFields="false"/>

  <fieldType name="bbox_ndv" class="solr.BBoxField"
             numberType="tdouble" distanceUnits="degrees" storeSubFields="false" docValues="false"/>

  <fieldType name="pbbox" class="solr.BBoxField"
             numberType="pdouble" distanceUnits="degrees" storeSubFields="false"/>

  <fieldType name="llp" class="solr.LatLonPointSpatialField" distanceUnits="degrees" multiValued="true" />
  <fieldType name="oslocation" class="solr.SpatialRecursivePrefixTreeFieldType" geo="false" maxDistErr="0.000009"
             worldBounds="ENVELOPE(0,700000,1300000,0)" distErrPct="0.15"/>
  <fieldType name="oslocationold" class="solr.SpatialRecursivePrefixTreeFieldType" geo="false" maxDistErr="0.000009"
             worldBounds="ENVELOPE(0,700000,1300000,0)" distErrPct="0.15" version="8.2.0"/>
  <field name="id" type="string" required="true"/>
  <field name="oslocation" type="oslocation" />
  <field name="oslocationold" type="oslocationold" />
  <field name="srpt_geohash" type="srpt_geohash" multiValued="true"/>
  <field name="srpt_quad" type="srpt_quad" multiValued="true"/>
  <field name="srpt_packedquad" type="srpt_packedquad" multiValued="true"/>
  <field name="srptgeom" type="srptgeom"/>
  <field name="srptgeom_geo3d" type="srptgeom_geo3d"/>
  <field name="srptgeom_s2_geo3d" type="srptgeom_s2_geo3d"/>
  <field name="bbox" type="bbox"/>
  <field name="pbbox" type="pbbox"/>
  <field name="bbox_ndv" type="bbox_ndv"/>
  <field name="llp" type="llp" indexed="true" docValues="true" />
  <field name="llp_idx" type="llp" indexed="true" docValues="false" />
  <field name="llp_dv" type="llp" indexed="false" docValues="true" />
  <field name="llp_1_dv_st" type="llp" indexed="false" docValues="true" stored="true" multiValued="false"/>
  <field name="llp_N_dv_st" type="llp" indexed="false" docValues="true" stored="true" multiValued="true"/>
  <field name="llp_1_dv" type="llp" indexed="false" docValues="true" stored="false" multiValued="false" useDocValuesAsStored="false"/>
  <field name="llp_N_dv" type="llp" indexed="false" docValues="true" stored="false" multiValued="true" useDocValuesAsStored="false"/>
  <field name="llp_1_dv_dvasst" type="llp" indexed="false" docValues="true" stored="false" multiValued="false"  useDocValuesAsStored="true"/>
  <field name="llp_N_dv_dvasst" type="llp" indexed="false" docValues="true" stored="false" multiValued="true"  useDocValuesAsStored="true"/>

  <dynamicField name="bboxD_*" type="bbox" indexed="true"/>
  <dynamicField name="str_*" type="string" indexed="true" stored="true"/>

  <!-- for DistanceFunctionTest -->
  <fieldType name="llp_km" class="solr.LatLonPointSpatialField" distanceUnits="kilometers" />
  <fieldType name="srpt_geohash_km" class="solr.SpatialRecursivePrefixTreeFieldType"
             geo="true" prefixTree="geohash" distanceUnits="kilometers"
  />
  <field name="srpt_geohash_km" type="srpt_geohash_km"/>
  <field name="llp_km" type="llp_km" docValues="true"/>
  <dynamicField name="*_td" type="tdouble" indexed="true" stored="true"/>
  <field name="point" type="point2D" indexed="true" stored="true" multiValued="false"/>

  <uniqueKey>id</uniqueKey>

</schema>


text:
namespace rocksdb {

static const int kDelayMicros = 100000;

class EnvPosixTest {
 private:
  port::Mutex mu_;
  std::string events_;

 public:
  Env* env_;
  EnvPosixTest() : env_(Env::Default()) { }
};

static void SetBool(void* ptr) {
  reinterpret_cast<port::AtomicPointer*>(ptr)->NoBarrier_Store(ptr);
}

TEST(EnvPosixTest, RunImmediately) {
  port::AtomicPointer called (nullptr);
  env_->Schedule(&SetBool, &called);
  Env::Default()->SleepForMicroseconds(kDelayMicros);
  ASSERT_TRUE(called.NoBarrier_Load() != nullptr);
}

TEST(EnvPosixTest, RunMany) {
  port::AtomicPointer last_id (nullptr);

  struct CB {
    port::AtomicPointer* last_id_ptr;   // Pointer to shared slot
    uintptr_t id;             // Order# for the execution of this callback

    CB(port::AtomicPointer* p, int i) : last_id_ptr(p), id(i) { }

    static void Run(void* v) {
      CB* cb = reinterpret_cast<CB*>(v);
      void* cur = cb->last_id_ptr->NoBarrier_Load();
      ASSERT_EQ(cb->id-1, reinterpret_cast<uintptr_t>(cur));
      cb->last_id_ptr->Release_Store(reinterpret_cast<void*>(cb->id));
    }
  };

  // Schedule in different order than start time
  CB cb1(&last_id, 1);
  CB cb2(&last_id, 2);
  CB cb3(&last_id, 3);
  CB cb4(&last_id, 4);
  env_->Schedule(&CB::Run, &cb1);
  env_->Schedule(&CB::Run, &cb2);
  env_->Schedule(&CB::Run, &cb3);
  env_->Schedule(&CB::Run, &cb4);

  Env::Default()->SleepForMicroseconds(kDelayMicros);
  void* cur = last_id.Acquire_Load();
  ASSERT_EQ(4U, reinterpret_cast<uintptr_t>(cur));
}

struct State {
  port::Mutex mu;
  int val;
  int num_running;
};

static void ThreadBody(void* arg) {
  State* s = reinterpret_cast<State*>(arg);
  s->mu.Lock();
  s->val += 1;
  s->num_running -= 1;
  s->mu.Unlock();
}

TEST(EnvPosixTest, StartThread) {
  State state;
  state.val = 0;
  state.num_running = 3;
  for (int i = 0; i < 3; i++) {
    env_->StartThread(&ThreadBody, &state);
  }
  while (true) {
    state.mu.Lock();
    int num = state.num_running;
    state.mu.Unlock();
    if (num == 0) {
      break;
    }
    Env::Default()->SleepForMicroseconds(kDelayMicros);
  }
  ASSERT_EQ(state.val, 3);
}

TEST(EnvPosixTest, TwoPools) {

  class CB {
   public:
    CB(const std::string& pool_name, int pool_size)
        : mu_(),
          num_running_(0),
          num_finished_(0),
          pool_size_(pool_size),
          pool_name_(pool_name) { }

    static void Run(void* v) {
      CB* cb = reinterpret_cast<CB*>(v);
      cb->Run();
    }

    void Run() {
      {
        MutexLock l(&mu_);
        num_running_++;
        std::cout << "Pool " << pool_name_ << ": "
                  << num_running_ << " running threads.\n";
        // make sure we don't have more than pool_size_ jobs running.
        ASSERT_LE(num_running_, pool_size_);
      }

      // sleep for 1 sec
      Env::Default()->SleepForMicroseconds(1000000);

      {
        MutexLock l(&mu_);
        num_running_--;
        num_finished_++;
      }
    }

    int NumFinished() {
      MutexLock l(&mu_);
      return num_finished_;
    }

   private:
    port::Mutex mu_;
    int num_running_;
    int num_finished_;
    int pool_size_;
    std::string pool_name_;
  };

  const int kLowPoolSize = 2;
  const int kHighPoolSize = 4;
  const int kJobs = 8;

  CB low_pool_job("low", kLowPoolSize);
  CB high_pool_job("high", kHighPoolSize);

  env_->SetBackgroundThreads(kLowPoolSize);
  env_->SetBackgroundThreads(kHighPoolSize, Env::Priority::HIGH);

  ASSERT_EQ(0U, env_->GetThreadPoolQueueLen(Env::Priority::LOW));
  ASSERT_EQ(0U, env_->GetThreadPoolQueueLen(Env::Priority::HIGH));

  // schedule same number of jobs in each pool
  for (int i = 0; i < kJobs; i++) {
    env_->Schedule(&CB::Run, &low_pool_job);
    env_->Schedule(&CB::Run, &high_pool_job, Env::Priority::HIGH);
  }
  // Wait a short while for the jobs to be dispatched.
  Env::Default()->SleepForMicroseconds(kDelayMicros);
  ASSERT_EQ((unsigned int)(kJobs - kLowPoolSize),
            env_->GetThreadPoolQueueLen());
  ASSERT_EQ((unsigned int)(kJobs - kLowPoolSize),
            env_->GetThreadPoolQueueLen(Env::Priority::LOW));
  ASSERT_EQ((unsigned int)(kJobs - kHighPoolSize),
            env_->GetThreadPoolQueueLen(Env::Priority::HIGH));

  // wait for all jobs to finish
  while (low_pool_job.NumFinished() < kJobs ||
         high_pool_job.NumFinished() < kJobs) {
    env_->SleepForMicroseconds(kDelayMicros);
  }

  ASSERT_EQ(0U, env_->GetThreadPoolQueueLen(Env::Priority::LOW));
  ASSERT_EQ(0U, env_->GetThreadPoolQueueLen(Env::Priority::HIGH));
}

#ifdef OS_LINUX
// To make sure the Env::GetUniqueId() related tests work correctly, The files
// should be stored in regular storage like "hard disk" or "flash device".
// Otherwise we cannot get the correct id.
//
// The following function act as the replacement of test::TmpDir() that may be
// customized by user to be on a storage that doesn't work with GetUniqueId().
//
// TODO(kailiu) This function still assumes /tmp/<test-dir> reside in regular
// storage system.
namespace {
bool IsSingleVarint(const std::string& s) {
  Slice slice(s);

  uint64_t v;
  if (!GetVarint64(&slice, &v)) {
    return false;
  }

  return slice.size() == 0;
}

bool IsUniqueIDValid(const std::string& s) {
  return !s.empty() && !IsSingleVarint(s);
}

const size_t MAX_ID_SIZE = 100;
char temp_id[MAX_ID_SIZE];

std::string GetOnDiskTestDir() {
  char base[100];
  snprintf(base, sizeof(base), "/tmp/rocksdbtest-%d",
           static_cast<int>(geteuid()));
  // Directory may already exist
  Env::Default()->CreateDirIfMissing(base);

  return base;
}
}  // namespace

// Only works in linux platforms
TEST(EnvPosixTest, RandomAccessUniqueID) {
  // Create file.
  const EnvOptions soptions;
  std::string fname = GetOnDiskTestDir() + "/" + "testfile";
  unique_ptr<WritableFile> wfile;
  ASSERT_OK(env_->NewWritableFile(fname, &wfile, soptions));

  unique_ptr<RandomAccessFile> file;

  // Get Unique ID
  ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions));
  size_t id_size = file->GetUniqueId(temp_id, MAX_ID_SIZE);
  ASSERT_TRUE(id_size > 0);
  std::string unique_id1(temp_id, id_size);
  ASSERT_TRUE(IsUniqueIDValid(unique_id1));

  // Get Unique ID again
  ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions));
  id_size = file->GetUniqueId(temp_id, MAX_ID_SIZE);
  ASSERT_TRUE(id_size > 0);
  std::string unique_id2(temp_id, id_size);
  ASSERT_TRUE(IsUniqueIDValid(unique_id2));

  // Get Unique ID again after waiting some time.
  env_->SleepForMicroseconds(1000000);
  ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions));
  id_size = file->GetUniqueId(temp_id, MAX_ID_SIZE);
  ASSERT_TRUE(id_size > 0);
  std::string unique_id3(temp_id, id_size);
  ASSERT_TRUE(IsUniqueIDValid(unique_id3));

  // Check IDs are the same.
  ASSERT_EQ(unique_id1, unique_id2);
  ASSERT_EQ(unique_id2, unique_id3);

  // Delete the file
  env_->DeleteFile(fname);
}

// only works in linux platforms
#ifdef ROCKSDB_FALLOCATE_PRESENT
TEST(EnvPosixTest, AllocateTest) {
  std::string fname = GetOnDiskTestDir() + "/preallocate_testfile";
  EnvOptions soptions;
  soptions.use_mmap_writes = false;
  unique_ptr<WritableFile> wfile;
  ASSERT_OK(env_->NewWritableFile(fname, &wfile, soptions));

  // allocate 100 MB
  size_t kPreallocateSize = 100 * 1024 * 1024;
  size_t kBlockSize = 512;
  std::string data = "test";
  wfile->SetPreallocationBlockSize(kPreallocateSize);
  ASSERT_OK(wfile->Append(Slice(data)));
  ASSERT_OK(wfile->Flush());

  struct stat f_stat;
  stat(fname.c_str(), &f_stat);
  ASSERT_EQ((unsigned int)data.size(), f_stat.st_size);
  // verify that blocks are preallocated
  // Note here that we don't check the exact number of blocks preallocated --
  // we only require that number of allocated blocks is at least what we expect.
  // It looks like some FS give us more blocks that we asked for. That's fine.
  // It might be worth investigating further.
  auto st_blocks = f_stat.st_blocks;
  ASSERT_LE((unsigned int)(kPreallocateSize / kBlockSize), st_blocks);

  // close the file, should deallocate the blocks
  wfile.reset();

  stat(fname.c_str(), &f_stat);
  ASSERT_EQ((unsigned int)data.size(), f_stat.st_size);
  // verify that preallocated blocks were deallocated on file close
  ASSERT_GT(st_blocks, f_stat.st_blocks);
}
#endif

// Returns true if any of the strings in ss are the prefix of another string.
bool HasPrefix(const std::unordered_set<std::string>& ss) {
  for (const std::string& s: ss) {
    if (s.empty()) {
      return true;
    }
    for (size_t i = 1; i < s.size(); ++i) {
      if (ss.count(s.substr(0, i)) != 0) {
        return true;
      }
    }
  }
  return false;
}

// Only works in linux platforms
TEST(EnvPosixTest, RandomAccessUniqueIDConcurrent) {
  // Check whether a bunch of concurrently existing files have unique IDs.
  const EnvOptions soptions;

  // Create the files
  std::vector<std::string> fnames;
  for (int i = 0; i < 1000; ++i) {
    fnames.push_back(GetOnDiskTestDir() + "/" + "testfile" + std::to_string(i));

    // Create file.
    unique_ptr<WritableFile> wfile;
    ASSERT_OK(env_->NewWritableFile(fnames[i], &wfile, soptions));
  }

  // Collect and check whether the IDs are unique.
  std::unordered_set<std::string> ids;
  for (const std::string fname: fnames) {
    unique_ptr<RandomAccessFile> file;
    std::string unique_id;
    ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions));
    size_t id_size = file->GetUniqueId(temp_id, MAX_ID_SIZE);
    ASSERT_TRUE(id_size > 0);
    unique_id = std::string(temp_id, id_size);
    ASSERT_TRUE(IsUniqueIDValid(unique_id));

    ASSERT_TRUE(ids.count(unique_id) == 0);
    ids.insert(unique_id);
  }

  // Delete the files
  for (const std::string fname: fnames) {
    ASSERT_OK(env_->DeleteFile(fname));
  }

  ASSERT_TRUE(!HasPrefix(ids));
}

// Only works in linux platforms
TEST(EnvPosixTest, RandomAccessUniqueIDDeletes) {
  const EnvOptions soptions;

  std::string fname = GetOnDiskTestDir() + "/" + "testfile";

  // Check that after file is deleted we don't get same ID again in a new file.
  std::unordered_set<std::string> ids;
  for (int i = 0; i < 1000; ++i) {
    // Create file.
    {
      unique_ptr<WritableFile> wfile;
      ASSERT_OK(env_->NewWritableFile(fname, &wfile, soptions));
    }

    // Get Unique ID
    std::string unique_id;
    {
      unique_ptr<RandomAccessFile> file;
      ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions));
      size_t id_size = file->GetUniqueId(temp_id, MAX_ID_SIZE);
      ASSERT_TRUE(id_size > 0);
      unique_id = std::string(temp_id, id_size);
    }

    ASSERT_TRUE(IsUniqueIDValid(unique_id));
    ASSERT_TRUE(ids.count(unique_id) == 0);
    ids.insert(unique_id);

    // Delete the file
    ASSERT_OK(env_->DeleteFile(fname));
  }

  ASSERT_TRUE(!HasPrefix(ids));
}

// Only works in linux platforms
TEST(EnvPosixTest, InvalidateCache) {
  const EnvOptions soptions;
  std::string fname = test::TmpDir() + "/" + "testfile";

  // Create file.
  {
    unique_ptr<WritableFile> wfile;
    ASSERT_OK(env_->NewWritableFile(fname, &wfile, soptions));
    ASSERT_OK(wfile.get()->Append(Slice("Hello world")));
    ASSERT_OK(wfile.get()->InvalidateCache(0, 0));
    ASSERT_OK(wfile.get()->Close());
  }

  // Random Read
  {
    unique_ptr<RandomAccessFile> file;
    char scratch[100];
    Slice result;
    ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions));
    ASSERT_OK(file.get()->Read(0, 11, &result, scratch));
    ASSERT_EQ(memcmp(scratch, "Hello world", 11), 0);
    ASSERT_OK(file.get()->InvalidateCache(0, 11));
    ASSERT_OK(file.get()->InvalidateCache(0, 0));
  }

  // Sequential Read
  {
    unique_ptr<SequentialFile> file;
    char scratch[100];
    Slice result;
    ASSERT_OK(env_->NewSequentialFile(fname, &file, soptions));
    ASSERT_OK(file.get()->Read(11, &result, scratch));
    ASSERT_EQ(memcmp(scratch, "Hello world", 11), 0);
    ASSERT_OK(file.get()->InvalidateCache(0, 11));
    ASSERT_OK(file.get()->InvalidateCache(0, 0));
  }
  // Delete the file
  ASSERT_OK(env_->DeleteFile(fname));
}
#endif

TEST(EnvPosixTest, PosixRandomRWFileTest) {
  EnvOptions soptions;
  soptions.use_mmap_writes = soptions.use_mmap_reads = false;
  std::string fname = test::TmpDir() + "/" + "testfile";

  unique_ptr<RandomRWFile> file;
  ASSERT_OK(env_->NewRandomRWFile(fname, &file, soptions));
  // If you run the unit test on tmpfs, then tmpfs might not
  // support fallocate. It is still better to trigger that
  // code-path instead of eliminating it completely.
  file.get()->Allocate(0, 10*1024*1024);
  ASSERT_OK(file.get()->Write(100, Slice("Hello world")));
  ASSERT_OK(file.get()->Write(105, Slice("Hello world")));
  ASSERT_OK(file.get()->Sync());
  ASSERT_OK(file.get()->Fsync());
  char scratch[100];
  Slice result;
  ASSERT_OK(file.get()->Read(100, 16, &result, scratch));
  ASSERT_EQ(result.compare("HelloHello world"), 0);
  ASSERT_OK(file.get()->Close());
}

class TestLogger : public Logger {
 public:
  virtual void Logv(const char* format, va_list ap) override {
    log_count++;

    char new_format[550];
    std::fill_n(new_format, sizeof(new_format), '2');
    {
      va_list backup_ap;
      va_copy(backup_ap, ap);
      int n = vsnprintf(new_format, sizeof(new_format) - 1, format, backup_ap);
      // 48 bytes for extra information + bytes allocated

      if (new_format[0] == '[') {
        // "[DEBUG] "
        ASSERT_TRUE(n <= 56 + (512 - static_cast<int>(sizeof(struct timeval))));
      } else {
        ASSERT_TRUE(n <= 48 + (512 - static_cast<int>(sizeof(struct timeval))));
      }
      va_end(backup_ap);
    }

    for (size_t i = 0; i < sizeof(new_format); i++) {
      if (new_format[i] == 'x') {
        char_x_count++;
      } else if (new_format[i] == '\0') {
        char_0_count++;
      }
    }
  }
  int log_count;
  int char_x_count;
  int char_0_count;
};

TEST(EnvPosixTest, LogBufferTest) {
  TestLogger test_logger;
  test_logger.SetInfoLogLevel(InfoLogLevel::INFO_LEVEL);
  test_logger.log_count = 0;
  test_logger.char_x_count = 0;
  test_logger.char_0_count = 0;
  LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, &test_logger);
  LogBuffer log_buffer_debug(DEBUG_LEVEL, &test_logger);

  char bytes200[200];
  std::fill_n(bytes200, sizeof(bytes200), '1');
  bytes200[sizeof(bytes200) - 1] = '\0';
  char bytes600[600];
  std::fill_n(bytes600, sizeof(bytes600), '1');
  bytes600[sizeof(bytes600) - 1] = '\0';
  char bytes9000[9000];
  std::fill_n(bytes9000, sizeof(bytes9000), '1');
  bytes9000[sizeof(bytes9000) - 1] = '\0';

  LogToBuffer(&log_buffer, "x%sx", bytes200);
  LogToBuffer(&log_buffer, "x%sx", bytes600);
  LogToBuffer(&log_buffer, "x%sx%sx%sx", bytes200, bytes200, bytes200);
  LogToBuffer(&log_buffer, "x%sx%sx", bytes200, bytes600);
  LogToBuffer(&log_buffer, "x%sx%sx", bytes600, bytes9000);

  LogToBuffer(&log_buffer_debug, "x%sx", bytes200);
  test_logger.SetInfoLogLevel(DEBUG_LEVEL);
  LogToBuffer(&log_buffer_debug, "x%sx%sx%sx", bytes600, bytes9000, bytes200);

  ASSERT_EQ(0, test_logger.log_count);
  log_buffer.FlushBufferToLog();
  log_buffer_debug.FlushBufferToLog();
  ASSERT_EQ(6, test_logger.log_count);
  ASSERT_EQ(6, test_logger.char_0_count);
  ASSERT_EQ(10, test_logger.char_x_count);
}

}  // namespace rocksdb

int main(int argc, char** argv) {
  return rocksdb::test::RunAllTests();
}


text:
package io.fsq.twofishes.country.test

import io.fsq.specs2.FSSpecificationWithJUnit
import io.fsq.twofishes.country.TimeZoneInfo
import org.specs2.matcher.MatchersImplicits

// TODO: See if there's a way to clean up the extra noise this sends to stderr.
class TimeZoneInfoSpec extends FSSpecificationWithJUnit with MatchersImplicits {
  "timezone lookup works" in {
    TimeZoneInfo.lookupTzID("America/Anchorage").map(_.cc) mustEqual Some("US")
  }

  "timezone lookup gracefully fails on missing" in {
    TimeZoneInfo.lookupTzID("America/Adsfdsgdfgsdg") mustEqual None
  }

  "timezone lookup throws on invalid CC" in {
    TimeZoneInfo.tzIdsFromIso2("USA") must throwAn[Exception]
  }

  "timezones lookup from country should work" in {
    TimeZoneInfo.tzIdsFromIso2("US") must containTheSameElementsAs(
      List(
        "America/Adak",
        "America/Anchorage",
        "America/Boise",
        "America/Chicago",
        "America/Denver",
        "America/Detroit",
        "America/Indiana/Indianapolis",
        "America/Indiana/Knox",
        "America/Indiana/Marengo",
        "America/Indiana/Petersburg",
        "America/Indiana/Tell_City",
        "America/Indiana/Vevay",
        "America/Indiana/Vincennes",
        "America/Indiana/Winamac",
        "America/Juneau",
        "America/Kentucky/Louisville",
        "America/Kentucky/Monticello",
        "America/Los_Angeles",
        "America/Menominee",
        "America/Metlakatla",
        "America/New_York",
        "America/Nome",
        "America/North_Dakota/Beulah",
        "America/North_Dakota/Center",
        "America/North_Dakota/New_Salem",
        "America/Phoenix",
        "America/Sitka",
        "America/Yakutat",
        "Pacific/Honolulu"
      )
    )

    TimeZoneInfo.tzIdsFromIso2("DE") must containTheSameElementsAs(
      List("Europe/Berlin", "Europe/Busingen")
    )
  }
}


text:
#!/bin/sh

VERSION=`./vsn.escript src/oblivion_boot.app.src`

mkdir /tmp/oblivion

cp -R * /tmp/oblivion

PWD_DIR=`pwd`
cd /tmp/oblivion

rm -f log/*
rm -f etc/persistence.data

cd ..

zip -r $PWD_DIR/oblivion-cache_${VERSION}.zip oblivion/bin oblivion/ebin oblivion/etc oblivion/log oblivion/lib/*/ebin oblivion/lib/*/include oblivion/lib/*/priv

rm -rf oblivion

cd $PWD_DIR


text:
package com.xonami.javaBells;

import org.jivesoftware.smack.XMPPConnection;
import org.jivesoftware.smack.packet.IQ;
import org.jivesoftware.smack.packet.IQ.Type;
import org.jivesoftware.smack.packet.PacketExtension;

import net.java.sip.communicator.impl.protocol.jabber.extensions.jingle.JingleIQ;
import net.java.sip.communicator.impl.protocol.jabber.extensions.jingle.JinglePacketFactory;
import net.java.sip.communicator.impl.protocol.jabber.extensions.jingle.Reason;

/**
 * This is a basic implementation of a JingleSession.
 * Without subclassing this, connections are rejected
 * and most other behavior is a reasonable default.
 *
 * @author bjorn
 *
 */
public class DefaultJingleSession implements JingleSession {
	public enum SessionState {
		NEW,
		NEGOTIATING_TRANSPORT,
		OPEN,
		CLOSED,
	}

	protected final JinglePacketHandler jinglePacketHandler;
	protected final String myJid;
	protected final String sessionId;
	protected final XMPPConnection connection;
	protected SessionState state;
	protected String peerJid;

	/** creates a new DefaultJingleSession with the given info. */
	public DefaultJingleSession( JinglePacketHandler jinglePacketHandler, String sessionId, XMPPConnection connection ) {
		this.jinglePacketHandler = jinglePacketHandler;
		this.myJid = connection.getUser();
		this.sessionId = sessionId;
		this.connection = connection;
		this.state = SessionState.NEW;
	}

	/** checks to make sure the packet came from the expected peer and that the state is not closed.
	 * If so, it acknowledges and returns true.
	 * If not, it returns false, sends a cancel, and sets the state to closed.
	 * Do NOT call this function before you set the peerJid.
	 */
	protected boolean checkAndAck( JingleIQ jiq ) {
		if( peerJid == null )
			throw new RuntimeException("Don't call this before setting peerJid!");
		if( state == SessionState.CLOSED )
			return false;
		if( peerJid.equals(jiq.getFrom()) ) {
			ack(jiq);
			return true;
		}
		closeSession(Reason.CONNECTIVITY_ERROR);
		return false;
	}

	protected boolean check( JingleIQ jiq ) {
		if( peerJid == null )
			throw new RuntimeException("Don't call this before setting peerJid!");
		if( state == SessionState.CLOSED )
			return false;
		if( peerJid.equals(jiq.getFrom()) ) {
			return true;
		}
		closeSession(Reason.CONNECTIVITY_ERROR);
		return false;
	}

	/** You may want to override this method to close any Jingle Streams you have open.
	 * To send a close message to the peer, include a reason. If reason is null, no message will be sent.*/
	protected void closeSession(Reason reason) {
		if( state == SessionState.CLOSED )
			return;
		if( reason != null )
			connection.sendPacket(JinglePacketFactory.createSessionTerminate(myJid, peerJid, sessionId, reason, null));
		state = SessionState.CLOSED;
		jinglePacketHandler.removeJingleSession(this);
	}

	/** Simply sends an ack to the given iq. */
	public void ack( IQ iq ) {
		IQ resp = IQ.createResultIQ(iq);
		connection.sendPacket(resp);
	}
	public void unsupportedInfo( IQ iq ) {
		IQ resp = IQ.createResultIQ(iq);
		resp.setType(Type.ERROR);
		resp.addExtension(new PacketExtension() {
			@Override
			public String getElementName() {
				return "error";
			}
			@Override
			public String getNamespace() {
				return null;
			}
			@Override
			public String toXML() {
				return "<error type='modify'><feature-not-implemented xmlns='urn:ietf:params:xml:ns:xmpp-stanzas'/><unsupported-info xmlns='urn:xmpp:jingle:errors:1'/></error>" ;
			}
		});
		connection.sendPacket(resp);
	}

	/** Calls checkAndAck. */
	public void handleContentAcept(JingleIQ jiq) {
		checkAndAck(jiq);
	}
	/** Calls checkAndAck. */
	public void handleContentAdd(JingleIQ jiq) {
		checkAndAck(jiq);
	}
	/** Calls checkAndAck. */
	public void handleContentModify(JingleIQ jiq) {
		checkAndAck(jiq);
	}
	/** Calls checkAndAck. */
	public void handleContentReject(JingleIQ jiq) {
		checkAndAck(jiq);
	}
	/** Calls checkAndAck. */
	public void handleContentRemove(JingleIQ jiq) {
		checkAndAck(jiq);
	}
	/** Calls checkAndAck. */
	public void handleDescriptionInfo(JingleIQ jiq) {
		checkAndAck(jiq);
	}
	/** Calls checkAndAck. */
	public void handleSecurityInfo(JingleIQ jiq) {
		checkAndAck(jiq);
	}
	/** Calls checkAndAck. */
	public void handleSessionAccept(JingleIQ jiq) {
		checkAndAck(jiq);
	}
	/** Calls checkAndAck. */
	public void handleSessionInfo(JingleIQ jiq) {
		checkAndAck(jiq);
	}

	/** sets the peerJid and closes the session. Subclasses will want to
	 * override this if they plan to handle incoming sessions. */
	public void handleSessionInitiate(JingleIQ jiq) {
		if( state == SessionState.CLOSED )
			return;
		ack(jiq);
		peerJid = jiq.getFrom();
		JingleIQ iq = JinglePacketFactory.createCancel(myJid, peerJid, sessionId);
		connection.sendPacket(iq);
		closeSession(Reason.DECLINE);
	}

	/** Closes the session. */
	public void handleSessionTerminate(JingleIQ jiq) {
		if( !checkAndAck(jiq) )
			return;
		closeSession(null);
	}
	/** Calls checkAndAck. */
	public void handleTransportAccept(JingleIQ jiq) {
		checkAndAck(jiq);
	}
	/** Calls checkAndAck. */
	public void handleTransportInfo(JingleIQ jiq) {
		checkAndAck(jiq);
	}
	/** Closes the session. */
	public void handleTransportReject(JingleIQ jiq) {
		if( !checkAndAck(jiq) )
			return;
		closeSession(Reason.GENERAL_ERROR);
	}
	/** Calls checkAndAck. */
	public void handleSessionReplace(JingleIQ jiq) {
		checkAndAck(jiq);
	}

	/** returns the sessionId for the current session. */
	@Override
	public String getSessionId() {
		return sessionId;
	}
}


text:
package testcase;

import annotations.MUTABLE;

public class AGT023_CharLiteral_ThisInstanceVariable {

	@MUTABLE
	public static AGT023_CharLiteral_ThisInstanceVariable test = new AGT023_CharLiteral_ThisInstanceVariable();

	public Object f = new Object();

	@Override
	public String toString() {
		return "AGT023_CharLiteral_ThisInstanceVariable [f=" + f + "]";
	}

	public void foo(){
		System.out.println(this.toString());
		this.f = '1';
		System.out.println(this.toString());
	}

	public static void main(String[] args) {
		test.foo();
	}

}


text:
package org.dmfs.rfc5545.recur;

import org.dmfs.rfc5545.Instance;
import org.dmfs.rfc5545.calendarmetrics.CalendarMetrics;
import org.dmfs.rfc5545.recur.RecurrenceRule.Skip;

/**
 * A filter that ensures invalid day of month numbers are skipped.
 *
 * @author Marten Gajda
 */
final class ByMonthDaySkipFilter extends RuleIterator
{
    /**
     * Stop iterating (throwing an exception) if this number of empty sets passed in a line, i.e. sets that contain no elements because they have been filtered
     * or nothing was expanded.
     */
    private final static int MAX_EMPTY_SETS = 1000;

    private final CalendarMetrics mCalendarMetrics;
    private final Skip mSkip;

    /**
     * The set we work on. This comes from the previous instance.
     */
    private LongArray mWorkingSet = null;

    /**
     * The set we return.
     */
    private final LongArray mResultSet = new LongArray();

    public ByMonthDaySkipFilter(RecurrenceRule rule, RuleIterator previous, CalendarMetrics calendarMetrics, long start)
    {
        super(previous);
        mCalendarMetrics = calendarMetrics;
        mSkip = rule.getSkip();
    }

    @Override
    public long next()
    {
        LongArray workingSet = mWorkingSet;
        if (workingSet == null || !workingSet.hasNext())
        {
            mWorkingSet = workingSet = nextSet();
        }
        return workingSet.next();
    }

    @Override
    LongArray nextSet()
    {
        LongArray resultSet = mResultSet;
        CalendarMetrics calendarMetrics = mCalendarMetrics;

        int counter = 0;
        do
        {
            if (counter == MAX_EMPTY_SETS)
            {
                throw new IllegalArgumentException("too many empty recurrence sets");
            }
            counter++;

            LongArray prev = mPrevious.nextSet();
            while (prev.hasNext())
            {
                long next = Instance.maskWeekday(prev.next());

                if (!calendarMetrics.validate(next))
                {
                    if (mSkip == Skip.BACKWARD)
                    {
                        next = calendarMetrics.prevDay(next);
                    }
                    else
                    // mSkip == Skip.FORWARD
                    {
                        next = calendarMetrics.nextDay(next);
                    }
                }
                resultSet.add(next);
            }
        } while (!resultSet.hasNext());

        return resultSet;
    }
}


text:
require 'rails_helper'

feature 'User sees existing todos' do
  let!(:todos) { create_pair(:todo) }

  scenario 'successfully' do
    visit admin_todos_path

    todos.each do |todo|
      expect(page).to have_selector('td', text: todo.id)
      expect(page).to have_selector('td', text: todo.title)
    end
  end
end


text:
describe('max', function()
  it('produces an error if its parent errors', function()
    local observable = Rx.Observable.of(''):map(function(x) return x() end)
    expect(observable).to.produce.error()
    expect(observable:max()).to.produce.error()
  end)

  it('produces an error if one of the values produced is a string', function()
    local observable = Rx.Observable.of(1, 'string'):max()
    expect(observable).to.produce.error()
  end)

  it('produces the maximum of all values produced', function()
    local observable = Rx.Observable.fromRange(5):max()
    expect(observable).to.produce(5)
  end)
end)


text:
package kesshou.android.daanx.util.network.api.holder;

import com.google.gson.annotations.SerializedName;

/*
   Author: Charles Lien(lienching)
   Description: Login Information JSON holder
*/
public class Login {

    @SerializedName("account")
    public String usr_account;

    @SerializedName("password")
    public String usr_password;

}


text:
package com.intellij.debugger.engine;

import com.intellij.debugger.jdi.ThreadReferenceProxyImpl;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.util.containers.SmartHashSet;
import com.sun.jdi.request.EventRequest;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.util.Set;

public class SuspendManagerUtil {
  private static final Logger LOG = Logger.getInstance("#com.intellij.debugger.engine.SuspendManagerUtil");

  public static boolean isEvaluating(SuspendManager suspendManager, ThreadReferenceProxyImpl thread) {
    for (SuspendContextImpl suspendContext : suspendManager.getEventContexts()) {
      if (suspendContext.isEvaluating() && thread.equals(suspendContext.getThread())) {
        return true;
      }
    }
    return false;
  }

  /**
   * Returns suspend context that suspends the thread specified (may be currently evaluating)
   */
  @Nullable
  public static SuspendContextImpl findContextByThread(@NotNull SuspendManager suspendManager, ThreadReferenceProxyImpl thread) {
    for (SuspendContextImpl context : ((SuspendManagerImpl)suspendManager).getPausedContexts()) {
      if ((context.getThread() == thread || context.getSuspendPolicy() == EventRequest.SUSPEND_ALL)
          && !context.isExplicitlyResumed(thread)){
        return context;
      }
    }

    return null;
  }

  public static void assertSuspendContext(SuspendContextImpl context) {
    if(LOG.isDebugEnabled()) {
      LOG.assertTrue(context.myInProgress, "You can invoke methods only inside commands invoked for SuspendContext");
    }
  }

  @NotNull
  public static Set<SuspendContextImpl> getSuspendingContexts(@NotNull SuspendManager suspendManager, ThreadReferenceProxyImpl thread) {
    DebuggerManagerThreadImpl.assertIsManagerThread();
    Set<SuspendContextImpl> result = new SmartHashSet<SuspendContextImpl>();
    for (SuspendContextImpl suspendContext : suspendManager.getEventContexts()) {
      if (suspendContext.suspends(thread)) {
        result.add(suspendContext);
      }
    }
    return result;
  }

  @Nullable
  public static SuspendContextImpl getSuspendingContext(@NotNull SuspendManager suspendManager, ThreadReferenceProxyImpl thread) {
    DebuggerManagerThreadImpl.assertIsManagerThread();
    for (SuspendContextImpl suspendContext : suspendManager.getEventContexts()) {
      if (suspendContext.suspends(thread)) {
        return suspendContext;
      }
    }
    return null;
  }

  public static void restoreAfterResume(SuspendContextImpl context, Object resumeData) {
    SuspendManager suspendManager = context.getDebugProcess().getSuspendManager();
    ResumeData data = (ResumeData) resumeData;

    ThreadReferenceProxyImpl thread = context.getThread();
    if(data.myIsFrozen && !suspendManager.isFrozen(thread)) {
      suspendManager.freezeThread(thread);
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("RestoreAfterResume SuspendContextImpl...");
    }
    LOG.assertTrue(context.myResumedThreads == null);

    if(data.myResumedThreads != null) {
      for (ThreadReferenceProxyImpl resumedThreads : data.myResumedThreads) {
        resumedThreads.resume();
      }
      context.myResumedThreads = data.myResumedThreads;
    }
  }

  public static Object prepareForResume(SuspendContextImpl context) {
    SuspendManager suspendManager = context.getDebugProcess().getSuspendManager();

    ThreadReferenceProxyImpl thread = context.getThread();

    ResumeData resumeData = new ResumeData(suspendManager.isFrozen(thread), context.myResumedThreads);

    if(resumeData.myIsFrozen) {
      suspendManager.unfreezeThread(thread);
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("Resuming SuspendContextImpl...");
    }
    if(context.myResumedThreads != null) {
      for (ThreadReferenceProxyImpl resumedThreads : context.myResumedThreads) {
        resumedThreads.suspend();
      }
      context.myResumedThreads = null;
    }

    return resumeData;
  }

  public static SuspendContextImpl getSuspendContextForThread(SuspendContextImpl suspendContext, ThreadReferenceProxyImpl thread) {
    if (suspendContext == null) {
      return null;
    }
    SuspendContextImpl context = findContextByThread(suspendContext.getDebugProcess().getSuspendManager(), thread);
    return context != null && !context.myInProgress ? context :  suspendContext;
  }

  public static SuspendContextImpl getEvaluatingContext(SuspendManager suspendManager, ThreadReferenceProxyImpl thread) {
    for (SuspendContextImpl suspendContext : suspendManager.getEventContexts()) {
      if (!suspendContext.isResumed() && suspendContext.isEvaluating() && suspendContext.getThread() == thread) {
        return suspendContext;
      }
    }
    return null;
  }

  private static class ResumeData {
    final boolean myIsFrozen;
    final Set<ThreadReferenceProxyImpl> myResumedThreads;

    public ResumeData(boolean isFrozen, Set<ThreadReferenceProxyImpl> resumedThreads) {
      myIsFrozen = isFrozen;
      myResumedThreads = resumedThreads;
    }
  }
}


text:
package org.apache.horn.core;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;

import org.apache.commons.lang.math.RandomUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hama.Constants;
import org.apache.hama.HamaConfiguration;
import org.apache.hama.bsp.BSPJob;
import org.apache.hama.commons.io.FloatMatrixWritable;
import org.apache.hama.commons.io.VectorWritable;
import org.apache.hama.commons.math.DenseFloatMatrix;
import org.apache.hama.commons.math.DenseFloatVector;
import org.apache.hama.commons.math.FloatFunction;
import org.apache.hama.commons.math.FloatMatrix;
import org.apache.hama.commons.math.FloatVector;
import org.apache.hama.util.ReflectionUtils;
import org.apache.horn.core.Constants.LearningStyle;
import org.apache.horn.core.Constants.TrainingMethod;
import org.apache.horn.examples.MultiLayerPerceptron.StandardNeuron;
import org.apache.horn.funcs.FunctionFactory;
import org.apache.horn.funcs.IdentityFunction;
import org.apache.horn.funcs.SoftMax;
import org.apache.horn.utils.MathUtils;

import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;

/**
 * SmallLayeredNeuralNetwork defines the general operations for derivative
 * layered models, include Linear Regression, Logistic Regression, Multilayer
 * Perceptron, Autoencoder, and Restricted Boltzmann Machine, etc. For
 * SmallLayeredNeuralNetwork, the training can be conducted in parallel, but the
 * parameters of the models are assumes to be stored in a single machine.
 *
 * In general, these models consist of neurons which are aligned in layers.
 * Between layers, for any two adjacent layers, the neurons are connected to
 * form a bipartite weighted graph.
 *
 */
public class LayeredNeuralNetwork extends AbstractLayeredNeuralNetwork {

  private static final Log LOG = LogFactory.getLog(LayeredNeuralNetwork.class);

  /* Weights between neurons at adjacent layers */
  protected List<FloatMatrix> weightMatrixList;

  /* Previous weight updates between neurons at adjacent layers */
  protected List<FloatMatrix> prevWeightUpdatesList;

  /* Different layers can have different squashing function */
  protected List<FloatFunction> squashingFunctionList;

  protected List<Class<? extends Neuron<?>>> neuronClassList;

  protected int finalLayerIdx;

  private List<Neuron<?>[]> neurons = new ArrayList<Neuron<?>[]>();

  private long iterations;

  public LayeredNeuralNetwork() {
    this.layerSizeList = Lists.newArrayList();
    this.weightMatrixList = Lists.newArrayList();
    this.prevWeightUpdatesList = Lists.newArrayList();
    this.squashingFunctionList = Lists.newArrayList();
    this.neuronClassList = Lists.newArrayList();
  }

  public LayeredNeuralNetwork(HamaConfiguration conf, String modelPath) {
    super(conf, modelPath);
    initializeNeurons(false);
  }

  public LayeredNeuralNetwork(HamaConfiguration conf, String modelPath,
      boolean isTraining) {
    super(conf, modelPath);
    initializeNeurons(isTraining);
  }

  // initialize neuron objects
  private void initializeNeurons(boolean isTraining) {
    for (int i = 0; i < layerSizeList.size(); i++) {
      int numOfNeurons = layerSizeList.get(i);
      Class<? extends Neuron<?>> neuronClass;
      if (i == 0)
        neuronClass = StandardNeuron.class; // actually doesn't needed
      else
        neuronClass = neuronClassList.get(i - 1);

      Neuron<?>[] tmp = new Neuron[numOfNeurons];
      for (int j = 0; j < numOfNeurons; j++) {
        Neuron<?> n = newNeuronInstance(neuronClass);
        if (i > 0)
          n.setSquashingFunction(squashingFunctionList.get(i - 1));
        else
          n.setSquashingFunction(new IdentityFunction());

        n.setLayerIndex(i);

        n.setNeuronID(j);
        n.setLearningRate(this.learningRate);
        n.setMomentumWeight(this.momentumWeight);
        n.setTraining(isTraining);
        tmp[j] = n;
      }

      neurons.add(tmp);
    }
  }

  @Override
  /**
   * {@inheritDoc}
   */
  public int addLayer(int size, boolean isFinalLayer,
      FloatFunction squashingFunction, Class<? extends Neuron<?>> neuronClass) {
    return addLayer(size, isFinalLayer, squashingFunction, neuronClass, null);
  }

  public int addLayer(int size, boolean isFinalLayer,
      FloatFunction squashingFunction, Class<? extends Neuron<?>> neuronClass,
      Class<? extends IntermediateOutput> interlayer) {
    Preconditions.checkArgument(size > 0,
        "Size of layer must be larger than 0.");
    if (!isFinalLayer) {
      if (this.layerSizeList.size() == 0) {
        LOG.info("add input layer: " + size + " neurons");
      } else {
        LOG.info("add hidden layer: " + size + " neurons");
      }
      size += 1;
    }

    this.layerSizeList.add(size);
    int layerIdx = this.layerSizeList.size() - 1;
    if (isFinalLayer) {
      this.finalLayerIdx = layerIdx;
      LOG.info("add output layer: " + size + " neurons");
    }

    // add weights between current layer and previous layer, and input layer has
    // no squashing function
    if (layerIdx > 0) {
      int sizePrevLayer = this.layerSizeList.get(layerIdx - 1);
      // row count equals to size of current size and column count equals to
      // size of previous layer
      int row = isFinalLayer ? size : size - 1;
      int col = sizePrevLayer;
      FloatMatrix weightMatrix = new DenseFloatMatrix(row, col);
      // initialize weights
      weightMatrix.applyToElements(new FloatFunction() {
        @Override
        public float apply(float value) {
          return RandomUtils.nextFloat() - 0.5f;
        }

        @Override
        public float applyDerivative(float value) {
          throw new UnsupportedOperationException("");
        }
      });
      this.weightMatrixList.add(weightMatrix);
      this.prevWeightUpdatesList.add(new DenseFloatMatrix(row, col));
      this.squashingFunctionList.add(squashingFunction);

      this.neuronClassList.add(neuronClass);
    }
    return layerIdx;
  }

  /**
   * Update the weight matrices with given matrices.
   *
   * @param matrices
   */
  public void updateWeightMatrices(FloatMatrix[] matrices) {
    for (int i = 0; i < matrices.length; ++i) {
      FloatMatrix matrix = this.weightMatrixList.get(i);
      this.weightMatrixList.set(i, matrix.add(matrices[i]));
    }
  }

  /**
   * Set the previous weight matrices.
   *
   * @param prevUpdates
   */
  void setPrevWeightMatrices(FloatMatrix[] prevUpdates) {
    this.prevWeightUpdatesList.clear();
    Collections.addAll(this.prevWeightUpdatesList, prevUpdates);
  }

  /**
   * Add a batch of matrices onto the given destination matrices.
   *
   * @param destMatrices
   * @param sourceMatrices
   */
  static void matricesAdd(FloatMatrix[] destMatrices,
      FloatMatrix[] sourceMatrices) {
    for (int i = 0; i < destMatrices.length; ++i) {
      destMatrices[i] = destMatrices[i].add(sourceMatrices[i]);
    }
  }

  /**
   * Get all the weight matrices.
   *
   * @return The matrices in form of matrix array.
   */
  FloatMatrix[] getWeightMatrices() {
    FloatMatrix[] matrices = new FloatMatrix[this.weightMatrixList.size()];
    this.weightMatrixList.toArray(matrices);
    return matrices;
  }

  /**
   * Set the weight matrices.
   *
   * @param matrices
   */
  public void setWeightMatrices(FloatMatrix[] matrices) {
    this.weightMatrixList = new ArrayList<FloatMatrix>();
    Collections.addAll(this.weightMatrixList, matrices);
  }

  /**
   * Get the previous matrices updates in form of array.
   *
   * @return The matrices in form of matrix array.
   */
  public FloatMatrix[] getPrevMatricesUpdates() {
    FloatMatrix[] prevMatricesUpdates = new FloatMatrix[this.prevWeightUpdatesList
        .size()];
    for (int i = 0; i < this.prevWeightUpdatesList.size(); ++i) {
      prevMatricesUpdates[i] = this.prevWeightUpdatesList.get(i);
    }
    return prevMatricesUpdates;
  }

  public void setWeightMatrix(int index, FloatMatrix matrix) {
    Preconditions.checkArgument(
        0 <= index && index < this.weightMatrixList.size(), String.format(
            "index [%d] should be in range[%d, %d].", index, 0,
            this.weightMatrixList.size()));
    this.weightMatrixList.set(index, matrix);
  }

  @Override
  public void readFields(DataInput input) throws IOException {
    super.readFields(input);

    this.finalLayerIdx = input.readInt();
    this.dropRate = input.readFloat();

    // read neuron classes
    int neuronClasses = input.readInt();
    this.neuronClassList = Lists.newArrayList();
    for (int i = 0; i < neuronClasses; ++i) {
      try {
        Class<? extends Neuron<?>> clazz = (Class<? extends Neuron<?>>) Class
            .forName(input.readUTF());
        neuronClassList.add(clazz);
      } catch (ClassNotFoundException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
    }

    // read squash functions
    int squashingFunctionSize = input.readInt();
    this.squashingFunctionList = Lists.newArrayList();
    for (int i = 0; i < squashingFunctionSize; ++i) {
      this.squashingFunctionList.add(FunctionFactory
          .createFloatFunction(WritableUtils.readString(input)));
    }

    // read weights and construct matrices of previous updates
    int numOfMatrices = input.readInt();
    this.weightMatrixList = Lists.newArrayList();
    this.prevWeightUpdatesList = Lists.newArrayList();
    for (int i = 0; i < numOfMatrices; ++i) {
      FloatMatrix matrix = FloatMatrixWritable.read(input);
      this.weightMatrixList.add(matrix);
      this.prevWeightUpdatesList.add(new DenseFloatMatrix(matrix.getRowCount(),
          matrix.getColumnCount()));
    }

  }

  @Override
  public void write(DataOutput output) throws IOException {
    super.write(output);
    output.writeInt(finalLayerIdx);
    output.writeFloat(dropRate);

    // write neuron classes
    output.writeInt(this.neuronClassList.size());
    for (Class<? extends Neuron<?>> clazz : this.neuronClassList) {
      output.writeUTF(clazz.getName());
    }

    // write squashing functions
    output.writeInt(this.squashingFunctionList.size());
    for (FloatFunction aSquashingFunctionList : this.squashingFunctionList) {
      WritableUtils.writeString(output,
          aSquashingFunctionList.getFunctionName());
    }

    // write weight matrices
    output.writeInt(this.weightMatrixList.size());
    for (FloatMatrix aWeightMatrixList : this.weightMatrixList) {
      FloatMatrixWritable.write(aWeightMatrixList, output);
    }

    // DO NOT WRITE WEIGHT UPDATE
  }

  @Override
  public FloatMatrix getWeightsByLayer(int layerIdx) {
    return this.weightMatrixList.get(layerIdx);
  }

  /**
   * Get the output of the model according to given feature instance.
   */
  @Override
  public FloatVector getOutput(FloatVector instance) {
    Preconditions.checkArgument(this.layerSizeList.get(0) - 1 == instance
        .getDimension(), String.format(
        "The dimension of input instance should be %d.",
        this.layerSizeList.get(0) - 1));
    // transform the features to another space
    FloatVector transformedInstance = this.featureTransformer
        .transform(instance);
    // add bias feature
    FloatVector instanceWithBias = new DenseFloatVector(
        transformedInstance.getDimension() + 1);
    instanceWithBias.set(0, 0.99999f); // set bias to be a little bit less than
                                       // 1.0
    for (int i = 1; i < instanceWithBias.getDimension(); ++i) {
      instanceWithBias.set(i, transformedInstance.get(i - 1));
    }

    // return the output of the last layer
    return getOutputInternal(instanceWithBias);
  }

  public void setDropRateOfInputLayer(float dropRate) {
    this.dropRate = dropRate;
  }

  /**
   * Calculate output internally, the intermediate output of each layer will be
   * stored.
   *
   * @param instanceWithBias The instance contains the features.
   * @return Cached output of each layer.
   */
  public FloatVector getOutputInternal(FloatVector instanceWithBias) {
    // sets the output of input layer
    Neuron<?>[] inputLayer = neurons.get(0);
    for (int i = 0; i < inputLayer.length; i++) {
      float m2 = MathUtils.getBinomial(1, dropRate);
      if(m2 == 0)
        inputLayer[i].setDrop(true);
      else
        inputLayer[i].setDrop(false);

      inputLayer[i].setOutput(instanceWithBias.get(i) * m2);
    }

    for (int i = 0; i < this.layerSizeList.size() - 1; ++i) {
      forward(i);
    }

    FloatVector output = new DenseFloatVector(
        neurons.get(this.finalLayerIdx).length);
    for (int i = 0; i < output.getDimension(); i++) {
      output.set(i, neurons.get(this.finalLayerIdx)[i].getOutput());
    }

    return output;
  }

  /**
   * @param neuronClass
   * @return a new neuron instance
   */
  @SuppressWarnings({ "rawtypes" })
  public static Neuron newNeuronInstance(Class<? extends Neuron> neuronClass) {
    return (Neuron) ReflectionUtils.newInstance(neuronClass);
  }

  public class InputMessageIterable implements
      Iterable<Synapse<FloatWritable, FloatWritable>> {
    private int currNeuronID;
    private int prevNeuronID;
    private int end;
    private FloatMatrix weightMat;
    private Neuron<?>[] layer;

    public InputMessageIterable(int fromLayer, int row) {
      this.currNeuronID = row;
      this.prevNeuronID = -1;
      this.end = weightMatrixList.get(fromLayer).getColumnCount() - 1;
      this.weightMat = weightMatrixList.get(fromLayer);
      this.layer = neurons.get(fromLayer);
    }

    @Override
    public Iterator<Synapse<FloatWritable, FloatWritable>> iterator() {
      return new MessageIterator();
    }

    private class MessageIterator implements
        Iterator<Synapse<FloatWritable, FloatWritable>> {

      @Override
      public boolean hasNext() {
        if (prevNeuronID < end) {
          return true;
        } else {
          return false;
        }
      }

      private FloatWritable i = new FloatWritable();
      private FloatWritable w = new FloatWritable();
      private Synapse<FloatWritable, FloatWritable> msg = new Synapse<FloatWritable, FloatWritable>();

      @Override
      public Synapse<FloatWritable, FloatWritable> next() {
        prevNeuronID++;

        i.set(layer[prevNeuronID].getOutput());
        w.set(weightMat.get(currNeuronID, prevNeuronID));
        msg.set(prevNeuronID, i, w);
        return new Synapse<FloatWritable, FloatWritable>(prevNeuronID, i, w);
      }

      @Override
      public void remove() {
      }

    }
  }

  /**
   * Forward the calculation for one layer.
   *
   * @param fromLayer The index of the previous layer.
   */
  protected void forward(int fromLayer) {
    int curLayerIdx = fromLayer + 1;
    FloatMatrix weightMatrix = this.weightMatrixList.get(fromLayer);

    FloatFunction squashingFunction = getSquashingFunction(fromLayer);
    FloatVector vec = new DenseFloatVector(weightMatrix.getRowCount());

    for (int row = 0; row < weightMatrix.getRowCount(); row++) {
      Neuron<?> n;
      if (curLayerIdx == finalLayerIdx)
        n = neurons.get(curLayerIdx)[row];
      else
        n = neurons.get(curLayerIdx)[row + 1];

      try {
        Iterable msgs = new InputMessageIterable(fromLayer, row);
        n.setIterationNumber(iterations);
        n.forward(msgs);
      } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
      vec.set(row, n.getOutput());
    }

    if (squashingFunction.getFunctionName().equalsIgnoreCase(
        SoftMax.class.getSimpleName())) {
      IntermediateOutput interlayer = (IntermediateOutput) ReflectionUtils
          .newInstance(SoftMax.SoftMaxOutputComputer.class);
      try {
        vec = interlayer.interlayer(vec);

        for (int i = 0; i < vec.getDimension(); i++) {
          neurons.get(curLayerIdx)[i].setOutput(vec.get(i));
        }

      } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
    }

    // add bias
    if (curLayerIdx != finalLayerIdx)
      neurons.get(curLayerIdx)[0].setOutput(1);
  }

  /**
   * Train the model online.
   *
   * @param trainingInstance
   */
  public void trainOnline(FloatVector trainingInstance) {
    FloatMatrix[] updateMatrices = this.trainByInstance(trainingInstance);
    this.updateWeightMatrices(updateMatrices);
  }

  @Override
  public FloatMatrix[] trainByInstance(FloatVector trainingInstance) {
    FloatVector transformedVector = this.featureTransformer
        .transform(trainingInstance.sliceUnsafe(this.layerSizeList.get(0) - 1));

    int inputDimension = this.layerSizeList.get(0) - 1;
    int outputDimension;
    FloatVector inputInstance = null;
    FloatVector labels = null;
    if (this.learningStyle == LearningStyle.SUPERVISED) {
      outputDimension = this.layerSizeList.get(this.layerSizeList.size() - 1);
      // validate training instance
      Preconditions.checkArgument(
          inputDimension + outputDimension == trainingInstance.getDimension(),
          String
              .format(
                  "The dimension of training instance is %d, but requires %d.",
                  trainingInstance.getDimension(), inputDimension
                      + outputDimension));

      inputInstance = new DenseFloatVector(this.layerSizeList.get(0));
      inputInstance.set(0, 1); // add bias
      // get the features from the transformed vector
      for (int i = 0; i < inputDimension; ++i) {
        inputInstance.set(i + 1, transformedVector.get(i));
      }
      // get the labels from the original training instance
      labels = trainingInstance.sliceUnsafe(inputInstance.getDimension() - 1,
          trainingInstance.getDimension() - 1);
    } else if (this.learningStyle == LearningStyle.UNSUPERVISED) {
      // labels are identical to input features
      outputDimension = inputDimension;
      // validate training instance
      Preconditions.checkArgument(inputDimension == trainingInstance
          .getDimension(), String.format(
          "The dimension of training instance is %d, but requires %d.",
          trainingInstance.getDimension(), inputDimension));

      inputInstance = new DenseFloatVector(this.layerSizeList.get(0));
      inputInstance.set(0, 1); // add bias
      // get the features from the transformed vector
      for (int i = 0; i < inputDimension; ++i) {
        inputInstance.set(i + 1, transformedVector.get(i));
      }
      // get the labels by copying the transformed vector
      labels = transformedVector.deepCopy();
    }

    FloatVector output = this.getOutputInternal(inputInstance);

    // get the training error
    calculateTrainingError(labels, output);

    if (this.trainingMethod.equals(TrainingMethod.GRADIENT_DESCENT)) {
      FloatMatrix[] updates = this.trainByInstanceGradientDescent(labels);
      return updates;
    } else {
      throw new IllegalArgumentException(
          String.format("Training method is not supported."));
    }
  }

  /**
   * Train by gradient descent. Get the updated weights using one training
   * instance.
   *
   * @param trainingInstance
   * @return The weight update matrices.
   */
  private FloatMatrix[] trainByInstanceGradientDescent(FloatVector labels) {

    // initialize weight update matrices
    DenseFloatMatrix[] weightUpdateMatrices = new DenseFloatMatrix[this.weightMatrixList
        .size()];
    for (int m = 0; m < weightUpdateMatrices.length; ++m) {
      weightUpdateMatrices[m] = new DenseFloatMatrix(this.weightMatrixList.get(
          m).getRowCount(), this.weightMatrixList.get(m).getColumnCount());
    }
    FloatVector deltaVec = new DenseFloatVector(
        this.layerSizeList.get(this.layerSizeList.size() - 1));

    FloatFunction squashingFunction = this.squashingFunctionList
        .get(this.squashingFunctionList.size() - 1);

    FloatMatrix lastWeightMatrix = this.weightMatrixList
        .get(this.weightMatrixList.size() - 1);

    for (int i = 0; i < deltaVec.getDimension(); ++i) {
      float finalOut = neurons.get(finalLayerIdx)[i].getOutput();
      float costFuncDerivative = this.costFunction.applyDerivative(
          labels.get(i), finalOut);
      // add regularization
      costFuncDerivative += this.regularizationWeight
          * lastWeightMatrix.getRowVector(i).sum();

      if (!squashingFunction.getFunctionName().equalsIgnoreCase(
          SoftMax.class.getSimpleName())) {
        costFuncDerivative *= squashingFunction.applyDerivative(finalOut);
      }

      neurons.get(finalLayerIdx)[i].backpropagate(costFuncDerivative);
      deltaVec.set(i, costFuncDerivative);
    }

    // start from previous layer of output layer
    for (int layer = this.layerSizeList.size() - 2; layer >= 0; --layer) {
      backpropagate(layer, weightUpdateMatrices[layer]);
    }

    this.setPrevWeightMatrices(weightUpdateMatrices);
    return weightUpdateMatrices;
  }

  public class ErrorMessageIterable implements
      Iterable<Synapse<FloatWritable, FloatWritable>> {
    private int row;
    private int neuronID;
    private int end;
    private FloatMatrix weightMat;
    private FloatMatrix prevWeightMat;

    private float[] nextLayerDelta;

    public ErrorMessageIterable(int curLayerIdx, int row) {
      this.row = row;
      this.neuronID = -1;
      this.weightMat = weightMatrixList.get(curLayerIdx);
      this.end = weightMat.getRowCount() - 1;
      this.prevWeightMat = prevWeightUpdatesList.get(curLayerIdx);

      Neuron<?>[] nextLayer = neurons.get(curLayerIdx + 1);
      nextLayerDelta = new float[weightMat.getRowCount()];

      for(int i = 0; i <= end; ++i) {
        if (curLayerIdx + 1 == finalLayerIdx) {
          nextLayerDelta[i] = nextLayer[i].getDelta();
        } else {
          nextLayerDelta[i] = nextLayer[i + 1].getDelta();
        }
      }
    }

    @Override
    public Iterator<Synapse<FloatWritable, FloatWritable>> iterator() {
      return new MessageIterator();
    }

    private class MessageIterator implements
        Iterator<Synapse<FloatWritable, FloatWritable>> {

      @Override
      public boolean hasNext() {
        if (neuronID < end) {
          return true;
        } else {
          return false;
        }
      }

      private FloatWritable d = new FloatWritable();
      private FloatWritable w = new FloatWritable();
      private FloatWritable p = new FloatWritable();
      private Synapse<FloatWritable, FloatWritable> msg = new Synapse<FloatWritable, FloatWritable>();

      @Override
      public Synapse<FloatWritable, FloatWritable> next() {
        neuronID++;

        d.set(nextLayerDelta[neuronID]);
        w.set(weightMat.get(neuronID, row));
        p.set(prevWeightMat.get(neuronID, row));
        msg.set(neuronID, d, w, p);
        return msg;
      }

      @Override
      public void remove() {
      }

    }
  }

  /**
   * Back-propagate the errors to from next layer to current layer. The weight
   * updated information will be stored in the weightUpdateMatrices, and the
   * delta of the prevLayer would be returned.
   *
   * @param layer Index of current layer.
   */
  private void backpropagate(int curLayerIdx,
  // FloatVector nextLayerDelta, FloatVector curLayerOutput,
      DenseFloatMatrix weightUpdateMatrix) {

    // get layer related information
    int x = this.weightMatrixList.get(curLayerIdx).getColumnCount();
    int y = this.weightMatrixList.get(curLayerIdx).getRowCount();

    FloatVector deltaVector = new DenseFloatVector(x);
    Neuron<?>[] ns = neurons.get(curLayerIdx);

    for (int row = 0; row < x; ++row) {
      Neuron<?> n = ns[row];
      n.setWeightVector(y);

      try {
        Iterable msgs = new ErrorMessageIterable(curLayerIdx, row);
        n.backward(msgs);
      } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }

      // update weights
      weightUpdateMatrix.setColumn(row, n.getWeights());
      deltaVector.set(row, n.getDelta());
    }

  }

  @Override
  protected BSPJob trainInternal(HamaConfiguration conf) throws IOException,
      InterruptedException, ClassNotFoundException {
    this.conf = conf;
    this.fs = FileSystem.get(conf);

    String modelPath = conf.get("model.path");
    if (modelPath != null) {
      this.modelPath = modelPath;
    }
    // modelPath must be set before training
    if (this.modelPath == null) {
      throw new IllegalArgumentException(
          "Please specify the modelPath for model, "
              + "either through setModelPath() or add 'modelPath' to the training parameters.");
    }
    this.writeModelToFile();

    // create job
    BSPJob job = new BSPJob(conf, LayeredNeuralNetworkTrainer.class);
    job.setJobName("Neural Network training");
    job.setJarByClass(LayeredNeuralNetworkTrainer.class);
    job.setBspClass(LayeredNeuralNetworkTrainer.class);

    job.getConfiguration().setInt(Constants.ADDITIONAL_BSP_TASKS, 1);

    job.setBoolean("training.mode", true);
    job.setInputPath(new Path(conf.get("training.input.path")));
    job.setInputFormat(org.apache.hama.bsp.SequenceFileInputFormat.class);
    job.setInputKeyClass(LongWritable.class);
    job.setInputValueClass(VectorWritable.class);
    job.setOutputKeyClass(NullWritable.class);
    job.setOutputValueClass(NullWritable.class);
    job.setOutputFormat(org.apache.hama.bsp.NullOutputFormat.class);

    return job;
  }

  @Override
  protected void calculateTrainingError(FloatVector labels, FloatVector output) {
    FloatVector errors = labels.deepCopy().applyToElements(output,
        this.costFunction);
    this.trainingError = errors.sum();
  }

  /**
   * Get the squashing function of a specified layer.
   *
   * @param idx
   * @return a new vector with the result of the operation.
   */
  public FloatFunction getSquashingFunction(int idx) {
    return this.squashingFunctionList.get(idx);
  }

  public void setIterationNumber(long iterations) {
    this.iterations = iterations;
  }

}