<|start_filename|>lyra2/cuda_lyra2h.cu<|end_filename|>
/**
 * Lyra2 (v1) cuda implementation based on djm34 work
 * tpruvot@github 2015, Nanashi 08/2016 (from 1.8-r2)
 * Lyra2h implentation for HPP coin based on all the previous
 * djm34 2017
 **/

#include <stdio.h>
#include <memory.h>

#define TPB52 8
#define TPB30 160
#define TPB20 160


//#include "cuda_lyra2h_sm2.cuh"
#include "cuda_lyra2h_sm5.cuh"

#ifdef __INTELLISENSE__
/* just for vstudio code colors */
#endif
static uint32_t *h_GNonces[16]; // this need to get fixed as the rest of that routine
static uint32_t *d_GNonces[16];

__constant__ uint32_t pTarget[8];

__constant__ static uint2 blake2b_IV_sm2[8] = {
	{ 0xf3bcc908, 0x6a09e667 },
	{ 0x84caa73b, 0xbb67ae85 },
	{ 0xfe94f82b, 0x3c6ef372 },
	{ 0x5f1d36f1, 0xa54ff53a },
	{ 0xade682d1, 0x510e527f },
	{ 0x2b3e6c1f, 0x9b05688c },
	{ 0xfb41bd6b, 0x1f83d9ab },
	{ 0x137e2179, 0x5be0cd19 }
};

#define reduceDuplexRow(rowIn, rowInOut, rowOut) { \
	for (int i = 0; i < 16; i++) { \
		for (int j = 0; j < 12; j++) \
			state[j] ^= Matrix[12 * i + j][rowIn] + Matrix[12 * i + j][rowInOut]; \
		round_lyra_sm2(state); \
		for (int j = 0; j < 12; j++) \
			Matrix[j + 12 * i][rowOut] ^= state[j]; \
		Matrix[0 + 12 * i][rowInOut] ^= state[11]; \
		Matrix[1 + 12 * i][rowInOut] ^= state[0]; \
		Matrix[2 + 12 * i][rowInOut] ^= state[1]; \
		Matrix[3 + 12 * i][rowInOut] ^= state[2]; \
		Matrix[4 + 12 * i][rowInOut] ^= state[3]; \
		Matrix[5 + 12 * i][rowInOut] ^= state[4]; \
		Matrix[6 + 12 * i][rowInOut] ^= state[5]; \
		Matrix[7 + 12 * i][rowInOut] ^= state[6]; \
		Matrix[8 + 12 * i][rowInOut] ^= state[7]; \
		Matrix[9 + 12 * i][rowInOut] ^= state[8]; \
		Matrix[10+ 12 * i][rowInOut] ^= state[9]; \
		Matrix[11+ 12 * i][rowInOut] ^= state[10]; \
	} \
  }

#define absorbblock(in)  { \
	state[0] ^= Matrix[0][in]; \
	state[1] ^= Matrix[1][in]; \
	state[2] ^= Matrix[2][in]; \
	state[3] ^= Matrix[3][in]; \
	state[4] ^= Matrix[4][in]; \
	state[5] ^= Matrix[5][in]; \
	state[6] ^= Matrix[6][in]; \
	state[7] ^= Matrix[7][in]; \
	state[8] ^= Matrix[8][in]; \
	state[9] ^= Matrix[9][in]; \
	state[10] ^= Matrix[10][in]; \
	state[11] ^= Matrix[11][in]; \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
  }


__device__ __forceinline__
static void round_lyra_sm2(uint2 *s)
{
	Gfunc(s[0], s[4], s[8], s[12]);
	Gfunc(s[1], s[5], s[9], s[13]);
	Gfunc(s[2], s[6], s[10], s[14]);
	Gfunc(s[3], s[7], s[11], s[15]);
	Gfunc(s[0], s[5], s[10], s[15]);
	Gfunc(s[1], s[6], s[11], s[12]);
	Gfunc(s[2], s[7], s[8], s[13]);
	Gfunc(s[3], s[4], s[9], s[14]);
}

__device__ __forceinline__
void reduceDuplexRowSetuph(const int rowIn, const int rowInOut, const int rowOut, uint2 state[16], uint2 Matrix[192][16])
{
#if __CUDA_ARCH__ > 500
#pragma unroll
#endif
	for (int i = 0; i < 16; i++)
	{
#pragma unroll
		for (int j = 0; j < 12; j++)
			state[j] ^= Matrix[12 * i + j][rowIn] + Matrix[12 * i + j][rowInOut];

		round_lyra_sm2(state);

#pragma unroll
		for (int j = 0; j < 12; j++)
			Matrix[j + 180 - 12 * i][rowOut] = Matrix[12 * i + j][rowIn] ^ state[j];

		Matrix[0 + 12 * i][rowInOut] ^= state[11];
		Matrix[1 + 12 * i][rowInOut] ^= state[0];
		Matrix[2 + 12 * i][rowInOut] ^= state[1];
		Matrix[3 + 12 * i][rowInOut] ^= state[2];
		Matrix[4 + 12 * i][rowInOut] ^= state[3];
		Matrix[5 + 12 * i][rowInOut] ^= state[4];
		Matrix[6 + 12 * i][rowInOut] ^= state[5];
		Matrix[7 + 12 * i][rowInOut] ^= state[6];
		Matrix[8 + 12 * i][rowInOut] ^= state[7];
		Matrix[9 + 12 * i][rowInOut] ^= state[8];
		Matrix[10 + 12 * i][rowInOut] ^= state[9];
		Matrix[11 + 12 * i][rowInOut] ^= state[10];
	}
}



__global__ __launch_bounds__(256, 1)
void lyra2h_gpu_hash_32_sm2(uint32_t threads, uint32_t startNounce, uint64_t *g_hash, uint32_t *resNonces)
{
	uint32_t thread = (blockDim.x * blockIdx.x + threadIdx.x);
	const uint2 Mask[8] = {
		{ 0x00000020, 0x00000000 },{ 0x00000020, 0x00000000 },
		{ 0x00000020, 0x00000000 },{ 0x00000010, 0x00000000 },
		{ 0x00000010, 0x00000000 },{ 0x00000010, 0x00000000 },
		{ 0x00000080, 0x00000000 },{ 0x00000000, 0x01000000 }
	};
	if (thread < threads)
	{
		uint2 state[16];

#pragma unroll
		for (int i = 0; i<4; i++) {
			LOHI(state[i].x, state[i].y, g_hash[threads*i + thread]);
		} //password

#pragma unroll
		for (int i = 0; i<4; i++) {
			state[i + 4] = state[i];
		} //salt

#pragma unroll
		for (int i = 0; i<8; i++) {
			state[i + 8] = blake2b_IV_sm2[i];
		}

		// blake2blyra x2
		//#pragma unroll 24
		for (int i = 0; i<12; i++) {
			round_lyra_sm2(state);
		}

		for (int i = 0; i<8; i++)
			state[i] ^= Mask[i];


		for (int i = 0; i<12; i++) {
			round_lyra_sm2(state);
		}


		uint2 Matrix[192][16]; // not cool

							 // reducedSqueezeRow0
#pragma unroll 16
		for (int i = 0; i < 16; i++)
		{
#pragma unroll 12
			for (int j = 0; j<12; j++) {
				Matrix[j + 180 - 12 * i][0] = state[j];
			}
			round_lyra_sm2(state);
		}

		// reducedSqueezeRow1
#pragma unroll 16
		for (int i = 0; i < 16; i++)
		{
#pragma unroll 12
			for (int j = 0; j<12; j++) {
				state[j] ^= Matrix[j + 12 * i][0];
			}
			round_lyra_sm2(state);
#pragma unroll 12
			for (int j = 0; j<12; j++) {
				Matrix[j + 180 - 12 * i][1] = Matrix[j + 12 * i][0] ^ state[j];
			}
		}

		reduceDuplexRowSetuph(1, 0, 2, state, Matrix);
		reduceDuplexRowSetuph(2, 1, 3, state, Matrix);
		reduceDuplexRowSetuph(3, 0, 4, state, Matrix);
		reduceDuplexRowSetuph(4, 3, 5, state, Matrix);
		reduceDuplexRowSetuph(5, 2, 6, state, Matrix);
		reduceDuplexRowSetuph(6, 1, 7, state, Matrix);
		reduceDuplexRowSetuph(7, 0, 8, state, Matrix);
		reduceDuplexRowSetuph(8, 3, 9, state, Matrix);
		reduceDuplexRowSetuph(9, 6, 10, state, Matrix);
		reduceDuplexRowSetuph(10, 1, 11, state, Matrix);
		reduceDuplexRowSetuph(11, 4, 12, state, Matrix);
		reduceDuplexRowSetuph(12, 7, 13, state, Matrix);
		reduceDuplexRowSetuph(13, 2, 14, state, Matrix);
		reduceDuplexRowSetuph(14, 5, 15, state, Matrix);

		uint32_t rowa;
		uint32_t prev = 15;
		uint32_t iterator = 0;

		for (int tau=0;tau<8;tau++) {
			for (uint32_t i = 0; i<16; i++) {
				rowa = state[0].x & 15;
				reduceDuplexRow(prev, rowa, iterator);
				prev = iterator;
				iterator = (iterator + 7) & 15;
			}
			for (uint32_t i = 0; i<16; i++) {
				rowa = state[0].x & 15;
				reduceDuplexRow(prev, rowa, iterator);
				prev = iterator;
				iterator = (iterator - 1) & 15;
			}
		}



		absorbblock(rowa);
		uint32_t nonce = startNounce + thread;
		if (((uint64_t*)state)[3] <= ((uint64_t*)pTarget)[3]) {
			atomicMin(&resNonces[1], resNonces[0]);
			atomicMin(&resNonces[0], nonce);
		}
	} //thread
}


__global__ __launch_bounds__(64, 1)
void lyra2h_gpu_hash_32_1(uint32_t threads, uint32_t startNounce, uint2 *g_hash)
{
	const uint32_t thread = (blockDim.x * blockIdx.x + threadIdx.x);
	const uint2x4 Mask[2] = {
		0x00000020UL, 0x00000000UL, 0x00000020UL, 0x00000000UL,
		0x00000020UL, 0x00000000UL, 0x00000010UL, 0x00000000UL,
		0x00000010UL, 0x00000000UL, 0x00000010UL, 0x00000000UL,
		0x00000080UL, 0x00000000UL, 0x00000000UL, 0x01000000UL
	};
	const uint2x4 blake2b_IV[2] = {
	0xf3bcc908lu, 0x6a09e667lu,
	0x84caa73blu, 0xbb67ae85lu,
	0xfe94f82blu, 0x3c6ef372lu,
	0x5f1d36f1lu, 0xa54ff53alu,
	0xade682d1lu, 0x510e527flu,
	0x2b3e6c1flu, 0x9b05688clu,
	0xfb41bd6blu, 0x1f83d9ablu,
	0x137e2179lu, 0x5be0cd19lu
    };
	if (thread < threads)
	{
		uint2x4 state[4];

		state[0].x = state[1].x = __ldg(&g_hash[thread + threads * 0]);
		state[0].y = state[1].y = __ldg(&g_hash[thread + threads * 1]);
		state[0].z = state[1].z = __ldg(&g_hash[thread + threads * 2]);
		state[0].w = state[1].w = __ldg(&g_hash[thread + threads * 3]);
		state[2] = blake2b_IV[0];
		state[3] = blake2b_IV[1];

		for (int i = 0; i<12; i++)
			round_lyra(state); //because 12 is not enough

		state[0] ^= Mask[0];
		state[1] ^= Mask[1];


		for (int i = 0; i<12; i++)
			round_lyra(state); //because 12 is not enough

		((uint2x4*)DMatrix)[threads * 0 + thread] = state[0];
		((uint2x4*)DMatrix)[threads * 1 + thread] = state[1];
		((uint2x4*)DMatrix)[threads * 2 + thread] = state[2];
		((uint2x4*)DMatrix)[threads * 3 + thread] = state[3];
	}
}

__global__
//__launch_bounds__(TPB52, 1)
void lyra2h_gpu_hash_32_2(uint32_t threads, uint32_t startNounce, uint64_t *g_hash)
{
	const uint32_t thread = blockDim.y * blockIdx.x + threadIdx.y;

	if (thread < threads)
	{
		uint2 state[4];
		state[0] = __ldg(&DMatrix[(0 * threads + thread) * blockDim.x + threadIdx.x]);
		state[1] = __ldg(&DMatrix[(1 * threads + thread) * blockDim.x + threadIdx.x]);
		state[2] = __ldg(&DMatrix[(2 * threads + thread) * blockDim.x + threadIdx.x]);
		state[3] = __ldg(&DMatrix[(3 * threads + thread) * blockDim.x + threadIdx.x]);

		reduceDuplexh2(state, thread, threads);
		reduceDuplexRowSetuph2(1, 0, 2, state, thread, threads);
		reduceDuplexRowSetuph2(2, 1, 3, state, thread, threads);
		reduceDuplexRowSetuph2(3, 0, 4, state, thread, threads);
		reduceDuplexRowSetuph2(4, 3, 5, state, thread, threads);
		reduceDuplexRowSetuph2(5, 2, 6, state, thread, threads);
		reduceDuplexRowSetuph2(6, 1, 7, state, thread, threads);
		reduceDuplexRowSetuph2(7, 0, 8, state, thread, threads);
		reduceDuplexRowSetuph2(8, 3, 9, state, thread, threads);
		reduceDuplexRowSetuph2(9, 6, 10, state, thread, threads);
		reduceDuplexRowSetuph2(10, 1, 11, state, thread, threads);
		reduceDuplexRowSetuph2(11, 4, 12, state, thread, threads);
		reduceDuplexRowSetuph2(12, 7, 13, state, thread, threads);
		reduceDuplexRowSetuph2(13, 2, 14, state, thread, threads);
		reduceDuplexRowSetuph2(14, 5, 15, state, thread, threads);

		uint32_t rowa; // = WarpShuffle(state[0].x, 0, 4) & 7;

		uint32_t prev = 15;
		uint32_t iterator = 0;


for (uint32_t tau=0;tau<7;tau++) {
		
		for (uint32_t i = 0; i<16; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 15;
			reduceDuplexRowth2(prev, rowa, iterator, state, thread, threads);
			prev = iterator;
			iterator = (iterator + 7) & 15;
		}

		for (uint32_t i = 0; i<16; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 15;
			reduceDuplexRowth2(prev, rowa, iterator, state, thread, threads);
			prev = iterator;
			iterator = (iterator - 1) & 15;
		}
}

		for (uint32_t i = 0; i<16; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 15;
			reduceDuplexRowth2(prev, rowa, iterator, state, thread, threads);
			prev = iterator;
			iterator = (iterator + 7) & 15;
		}

		for (uint32_t i = 0; i<15; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 15;
			reduceDuplexRowth2(prev, rowa, iterator, state, thread, threads);
			prev = iterator;
			iterator = (iterator - 1) & 15;
		}

//}
		rowa = WarpShuffle(state[0].x, 0, 4) & 15;
		reduceDuplexRowt_8_v2h(prev,iterator,rowa, state, thread, threads);


		DMatrix[(0 * threads + thread) * blockDim.x + threadIdx.x] = state[0];
		DMatrix[(1 * threads + thread) * blockDim.x + threadIdx.x] = state[1];
		DMatrix[(2 * threads + thread) * blockDim.x + threadIdx.x] = state[2];
		DMatrix[(3 * threads + thread) * blockDim.x + threadIdx.x] = state[3];
	}
}

__global__ __launch_bounds__(64, 1)
void lyra2h_gpu_hash_32_3(uint32_t threads, uint32_t startNounce, uint2 *g_hash, uint32_t *resNonces)
{
	const uint32_t thread = blockDim.x * blockIdx.x + threadIdx.x;

	uint28 state[4];

	if (thread < threads)
	{
		state[0] = __ldg4(&((uint2x4*)DMatrix)[threads * 0 + thread]);
		state[1] = __ldg4(&((uint2x4*)DMatrix)[threads * 1 + thread]);
		state[2] = __ldg4(&((uint2x4*)DMatrix)[threads * 2 + thread]);
		state[3] = __ldg4(&((uint2x4*)DMatrix)[threads * 3 + thread]);

		for (int i = 0; i < 12; i++)
			round_lyra(state);
		uint32_t nonce = startNounce + thread;
		if (((uint64_t*)state)[3] <= ((uint64_t*)pTarget)[3]) {
			atomicMin(&resNonces[1], resNonces[0]);
			atomicMin(&resNonces[0], nonce);
		}
/*
		g_hash[thread + threads * 0] = state[0].x;
		g_hash[thread + threads * 1] = state[0].y;
		g_hash[thread + threads * 2] = state[0].z;
		g_hash[thread + threads * 3] = state[0].w;
*/
	} //thread
}


__global__ __launch_bounds__(64, 1)
void lyra2h_gpu_hash_32_1_sm5(uint32_t threads, uint32_t startNounce, uint2 *g_hash)
{
	const uint32_t thread = (blockDim.x * blockIdx.x + threadIdx.x);

	const uint2x4 blake2b_IV[2] = {
		{ { 0xf3bcc908, 0x6a09e667 },{ 0x84caa73b, 0xbb67ae85 },{ 0xfe94f82b, 0x3c6ef372 },{ 0x5f1d36f1, 0xa54ff53a } },
		{ { 0xade682d1, 0x510e527f },{ 0x2b3e6c1f, 0x9b05688c },{ 0xfb41bd6b, 0x1f83d9ab },{ 0x137e2179, 0x5be0cd19 } }
	};
	const uint2x4 Mask[2] = {
		0x00000020UL, 0x00000000UL, 0x00000020UL, 0x00000000UL,
		0x00000020UL, 0x00000000UL, 0x00000010UL, 0x00000000UL,
		0x00000010UL, 0x00000000UL, 0x00000010UL, 0x00000000UL,
		0x00000080UL, 0x00000000UL, 0x00000000UL, 0x01000000UL
	};
	if (thread < threads)
	{
		uint2x4 state[4];

		((uint2*)state)[0] = __ldg(&g_hash[thread]);
		((uint2*)state)[1] = __ldg(&g_hash[thread + threads]);
		((uint2*)state)[2] = __ldg(&g_hash[thread + threads * 2]);
		((uint2*)state)[3] = __ldg(&g_hash[thread + threads * 3]);

		state[1] = state[0];
		state[2] = blake2b_IV[0];
		state[3] = blake2b_IV[1];

		for (int i = 0; i < 12; i++)
			round_lyra(state); //because 12 is not enough

		state[0] ^= Mask[0];
		state[1] ^= Mask[1];

		for (int i = 0; i < 12; i++)
			round_lyra(state); //because 12 is not enough


		((uint2x4*)DMatrix)[0 * threads + thread] = state[0];
		((uint2x4*)DMatrix)[1 * threads + thread] = state[1];
		((uint2x4*)DMatrix)[2 * threads + thread] = state[2];
		((uint2x4*)DMatrix)[3 * threads + thread] = state[3];
	}
}

__global__ __launch_bounds__(8, 1)
void lyra2h_gpu_hash_32_2_sm5(uint32_t threads, uint32_t startNounce, uint2 *g_hash)
{
	const uint32_t thread = (blockDim.y * blockIdx.x + threadIdx.y);

	if (thread < threads)
	{
		uint2 state[4];

		state[0] = __ldg(&DMatrix[(0 * threads + thread)*blockDim.x + threadIdx.x]);
		state[1] = __ldg(&DMatrix[(1 * threads + thread)*blockDim.x + threadIdx.x]);
		state[2] = __ldg(&DMatrix[(2 * threads + thread)*blockDim.x + threadIdx.x]);
		state[3] = __ldg(&DMatrix[(3 * threads + thread)*blockDim.x + threadIdx.x]);

		reduceDuplexV5h(state, thread, threads);

		uint32_t rowa; // = WarpShuffle(state[0].x, 0, 4) & 7;
		uint32_t prev = 15;
		uint32_t iterator = 0;
		for (int tau = 0; tau<7; tau++) {
			for (uint32_t i = 0; i<16; i++) {
				rowa = WarpShuffle(state[0].x, 0, 4) & 15;
				reduceDuplexRowV50h(prev, rowa, iterator, state, thread, threads);
				prev = iterator;
				iterator = (iterator + 7) & 15;
			}
			for (uint32_t i = 0; i<16; i++) {
				rowa = WarpShuffle(state[0].x, 0, 4) & 15;
				reduceDuplexRowV50h(prev, rowa, iterator, state, thread, threads);
				prev = iterator;
				iterator = (iterator - 1) & 15;
			}
		}

		for (uint32_t i = 0; i<16; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 15;
			reduceDuplexRowV50h(prev, rowa, iterator, state, thread, threads);
			prev = iterator;
			iterator = (iterator + 7) & 15;
		}
		for (uint32_t i = 0; i<15; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 15;
			reduceDuplexRowV50h(prev, rowa, iterator, state, thread, threads);
			prev = iterator;
			iterator = (iterator - 1) & 15;
		}

		rowa = WarpShuffle(state[0].x, 0, 4) & 15;
		reduceDuplexRowV50_8_v2h(prev, iterator, rowa, state, thread, threads);



		DMatrix[(0 * threads + thread)*blockDim.x + threadIdx.x] = state[0];
		DMatrix[(1 * threads + thread)*blockDim.x + threadIdx.x] = state[1];
		DMatrix[(2 * threads + thread)*blockDim.x + threadIdx.x] = state[2];
		DMatrix[(3 * threads + thread)*blockDim.x + threadIdx.x] = state[3];
	}
}

__global__ __launch_bounds__(64, 1)
void lyra2h_gpu_hash_32_3_sm5(uint32_t threads, uint32_t startNounce, uint2 *g_hash, uint32_t *resNonces)
{
	const uint32_t thread = (blockDim.x * blockIdx.x + threadIdx.x);

	if (thread < threads)
	{
		uint2x4 state[4];

		state[0] = __ldg4(&((uint2x4*)DMatrix)[0 * threads + thread]);
		state[1] = __ldg4(&((uint2x4*)DMatrix)[1 * threads + thread]);
		state[2] = __ldg4(&((uint2x4*)DMatrix)[2 * threads + thread]);
		state[3] = __ldg4(&((uint2x4*)DMatrix)[3 * threads + thread]);

		for (int i = 0; i < 12; i++)
			round_lyra(state);


		uint32_t nonce = startNounce + thread;
		if (((uint64_t*)state)[3] <= ((uint64_t*)pTarget)[3]) {
			atomicMin(&resNonces[1], resNonces[0]);
			atomicMin(&resNonces[0], nonce);
		}

	}
}




__global__	__launch_bounds__(48, 1)
void lyra2h_gpu_hash_32_v3(uint32_t threads, uint32_t startNounce, uint2 *outputHash, uint32_t *resNonces)
{
	uint32_t thread = (blockDim.x * blockIdx.x + threadIdx.x);

	ulonglong4 state[4];


	const ulonglong4 blake2b_IV[2] = {
		{ 0x6a09e667f3bcc908,
		0xbb67ae8584caa73b,
		0x3c6ef372fe94f82b,
		0xa54ff53a5f1d36f1 },
		{ 0x510e527fade682d1,
		0x9b05688c2b3e6c1f,
		0x1f83d9abfb41bd6b,
		0x5be0cd19137e2179 } };

	const ulonglong4 Mask[2] = {
		{ 0x20,
		0x20,
		0x20,
		0x08 },
		{ 0x08,
		0x08,
		0x80,
		0x0100000000000000 }
	};

#if __CUDA_ARCH__ == 350 || __CUDA_ARCH__ == 370
	if (thread < threads)
#endif
	{

		((uint2*)state)[0] = __ldg(&outputHash[thread]);
		((uint2*)state)[1] = __ldg(&outputHash[thread + threads]);
		((uint2*)state)[2] = __ldg(&outputHash[thread + threads * 2]);
		((uint2*)state)[3] = __ldg(&outputHash[thread + threads * 3]);

		state[1] = state[0];

		state[2] = ((ulonglong4*)blake2b_IV)[0];
		state[3] = ((ulonglong4*)blake2b_IV)[1];

		for (int i = 0; i<12; i++)
			round_lyra_v35(state);  //because 12 is not enough

		state[0] ^= Mask[0];
		state[1] ^= Mask[1];

		for (int i = 0; i<12; i++)
			round_lyra_v35(state);  //because 12 is not enough


		uint32_t ps1 = (8 * memshift * 7 + 64 * memshift * thread);


		for (int i = 0; i < 8; i++)
		{
			uint32_t s1 = ps1 - 8 * memshift * i;
			for (int j = 0; j < 3; j++)
				((ulonglong4*)(DMatrix35))[j + s1] = (state)[j];

			round_lyra_v35(state);
		}


		reduceDuplexV3(state, thread);

		reduceDuplexRowSetupV3(1, 0, 2, state, thread);
		reduceDuplexRowSetupV3(2, 1, 3, state, thread);
		reduceDuplexRowSetupV3(3, 0, 4, state, thread);
		reduceDuplexRowSetupV3(4, 3, 5, state, thread);
		reduceDuplexRowSetupV3(5, 2, 6, state, thread);
		reduceDuplexRowSetupV3(6, 1, 7, state, thread);

		uint32_t rowa;
		uint32_t prev = 7;
		uint32_t iterator = 0;
		for (int j = 0; j<4; j++) {
			for (uint32_t i = 0; i<8; i++) {
				rowa = ((uint2*)state)[0].x & 7;
				reduceDuplexRowtV3(prev, rowa, iterator, state, thread);
				prev = iterator;
				iterator = (iterator + 3) & 7;
			}

			for (uint32_t i = 0; i<8; i++) {
				rowa = ((uint2*)state)[0].x & 7;
				reduceDuplexRowtV3(prev, rowa, iterator, state, thread);
				prev = iterator;
				iterator = (iterator - 1) & 7;
			}

		}

		uint32_t shift = (memshift * rowa + 64 * memshift * thread);

		for (int j = 0; j < 3; j++)
			state[j] ^= __ldg4(&((ulonglong4*)(DMatrix35))[j + shift]);

		for (int i = 0; i < 12; i++)
			round_lyra_v35(state);

		uint32_t nonce = startNounce + thread;
		if (((uint64_t*)state)[3] <= ((uint64_t*)pTarget)[3]) {
			atomicMin(&resNonces[1], resNonces[0]);
			atomicMin(&resNonces[0], nonce);
		}

	} //thread
}



__global__	__launch_bounds__(4 * 16, 1)
void lyra2h_gpu_hash_32_ws2(uint32_t threads, uint32_t startNounce, uint2 *outputHash, uint32_t *resNonces)
{
#if __CUDA_ARCH__ > 300 

	uint32_t thread = (blockDim.x * blockIdx.x + threadIdx.x);
	uint32_t thread2 = thread >> 2;

	vectype state;
	vectype temp[16];


	const uint28 blake2b_IV[2] = {
		{ 0xf3bcc908, 0x6a09e667,
		0x84caa73b, 0xbb67ae85,
		0xfe94f82b, 0x3c6ef372,
		0x5f1d36f1, 0xa54ff53a },
		{ 0xade682d1, 0x510e527f,
		0x2b3e6c1f, 0x9b05688c,
		0xfb41bd6b, 0x1f83d9ab,
		0x137e2179, 0x5be0cd19 } };

	const uint28 padding[2] = {
		{ 0x20, 0x0,
		0x20, 0x0,
		0x20, 0x0,
		0x10, 0x0 },
		{ 0x10, 0x0,
		0x10, 0x0,
		0x80, 0x0,
		0x00, 0x01000000 } };

	//		state.x = __ldg(&((u64type*)outputHash)[4 * thread2 + (threadIdx.x % 4)]);

	state.x = __ldg(&((u64type*)outputHash)[thread2 + (threadIdx.x % 4) * threads]);
	state.y = state.x;

	state.z = ((u64type*)blake2b_IV)[threadIdx.x % 4];
	state.w = ((u64type*)blake2b_IV)[4 + threadIdx.x % 4];


	for (int i = 0; i<12; i++)
		round_lyra_v35_ws(state);

	state.x ^= ((u64type*)padding)[threadIdx.x % 4];
	state.y ^= ((u64type*)padding)[4 + threadIdx.x % 4];


	for (int i = 0; i<12; i++)
		round_lyra_v35_ws(state);

	uint32_t ps1 = (memshift * (Ncol - 1) + Nrow * Ncol * memshift * thread2);

	for (int i = 0; i < Ncol; i++)
	{
		uint32_t s1 = ps1 - memshift * i;
		((vectype*)(DMatrix35))[(threadIdx.x % 4) + s1] = state;
		temp[(15 - i)] = state;
		round_lyra_v35_ws(state);
	}

	reduceDuplex_ws2(state, thread2, temp);
	reduceDuplexRowSetup_ws2_pass1(1, 0, 2, state, thread2, temp);
	reduceDuplexRowSetup_ws2_pass2(2, 1, 3, state, thread2, temp);
	reduceDuplexRowSetup_ws2_pass1(3, 0, 4, state, thread2, temp);
	reduceDuplexRowSetup_ws2_pass2(4, 3, 5, state, thread2, temp);
	reduceDuplexRowSetup_ws2_pass1(5, 2, 6, state, thread2, temp);
	reduceDuplexRowSetup_ws2_pass2(6, 1, 7, state, thread2, temp);
	reduceDuplexRowSetup_ws2_pass1(7, 0, 8, state, thread2, temp);
	reduceDuplexRowSetup_ws2_pass2(8, 3, 9, state, thread2, temp);
	reduceDuplexRowSetup_ws2_pass1(9, 6, 10, state, thread2, temp);
	reduceDuplexRowSetup_ws2_pass2(10, 1, 11, state, thread2, temp);
	reduceDuplexRowSetup_ws2_pass1(11, 4, 12, state, thread2, temp);
	reduceDuplexRowSetup_ws2_pass2(12, 7, 13, state, thread2, temp);
	reduceDuplexRowSetup_ws2_pass1(13, 2, 14, state, thread2, temp);
	reduceDuplexRowSetup_ws2_pass2(14, 5, 15, state, thread2, temp);

	uint32_t rowa;
	uint32_t prev = 15;
	uint32_t iterator = 0;
	for (int j = 0; j<8; j++) {
		for (int i = 0; i < Ncol; i++)
		{
			rowa = shuffle2t(state.x, 0, 4) & 15;
			reduceDuplexRow_ws2(prev, rowa, iterator, state, thread2, temp);
			prev = iterator;
			iterator = (iterator + 7) & 15;
		}
		for (int i = 0; i < Ncol; i++)
		{
			rowa = shuffle2t(state.x, 0, 4) & 15;
			if (i == 15 && j == 7)
				reduceDuplexRow_ws2_v2(prev, rowa, iterator, state, thread2, temp);
			else
				reduceDuplexRow_ws2(prev, rowa, iterator, state, thread2, temp);

			prev = iterator;
			iterator = (iterator - 1) & 15;
		}
	}

	uint32_t shift = (memshift * Ncol * rowa + Nrow * Ncol * memshift * thread2);
	vectype tmp2 = temp[15];//__ldg4t(&((vectype*)(DMatrix35))[(threadIdx.x % 4) + shift]);
	state.x ^= tmp2.x;
	state.y ^= tmp2.y;
	state.z ^= tmp2.z;

	for (int i = 0; i < 12; i++)
		round_lyra_v35_ws(state);

	//	((u64type*)outputHash)[4 * thread2 + (threadIdx.x % 4)] = state.x;

	if ((threadIdx.x % 4) == 3) {
		uint32_t nonce = startNounce + thread2;
		if (devectorize(state.x) <= ((uint64_t*)pTarget)[3]) {
			atomicMin(&resNonces[1], resNonces[0]);
			atomicMin(&resNonces[0], nonce);
		}
	}
#endif
}





__host__
void lyra2h_cpu_init(int thr_id, uint32_t threads, uint64_t *d_matrix)
{

	// just assign the device pointer allocated in main loop
	cudaMemcpyToSymbol(DMatrix, &d_matrix, sizeof(uint64_t*), 0, cudaMemcpyHostToDevice);
	cudaMalloc(&d_GNonces[thr_id], 2 * sizeof(uint32_t));
	cudaMallocHost(&h_GNonces[thr_id], 2 * sizeof(uint32_t));
}

__host__
void lyra2h_cpu_init_sm2(int thr_id, uint32_t threads)
{

	// just assign the device pointer allocated in main loop
	cudaMalloc(&d_GNonces[thr_id], 2 * sizeof(uint32_t));
	cudaMallocHost(&h_GNonces[thr_id], 2 * sizeof(uint32_t));
}
  

__host__
void lyra2h_cpu_init_sm35(int thr_id, uint32_t threads, uint64_t *d_matrix)
{

	// just assign the device pointer allocated in main loop
	cudaMemcpyToSymbol(DMatrix35, &d_matrix, sizeof(uint64_t*), 0, cudaMemcpyHostToDevice);
	cudaMalloc(&d_GNonces[thr_id], 2 * sizeof(uint32_t));
	cudaMallocHost(&h_GNonces[thr_id], 2 * sizeof(uint32_t));
}


__host__
uint32_t lyra2h_getSecNonce(int thr_id, int num)
{
	uint32_t results[2];
	memset(results, 0xFF, sizeof(results));
	cudaMemcpy(results, d_GNonces[thr_id], sizeof(results), cudaMemcpyDeviceToHost);
	if (results[1] == results[0])
		return UINT32_MAX;
	return results[num];
}

__host__
void lyra2h_setTarget(const void *pTargetIn)
{
	cudaMemcpyToSymbol(pTarget, pTargetIn, 32, 0, cudaMemcpyHostToDevice);
}


__host__
uint32_t lyra2h_cpu_hash_32(int thr_id, uint32_t threads, uint32_t startNounce, uint64_t *d_hash, bool gtx750ti)
{
	uint32_t result = UINT32_MAX;
	cudaMemset(d_GNonces[thr_id], 0xff, 2 * sizeof(uint32_t));
	int dev_id = device_map[thr_id % MAX_GPUS];

	uint32_t tpb = TPB52;

	if (device_sm[dev_id] == 500) 
		tpb = TPB52;
	if (device_sm[dev_id] == 200) 
		tpb = TPB52;
	
	uint32_t tpb3 = 256;

	dim3 grid1((threads * 4 + tpb - 1) / tpb);
	dim3 block1(4, tpb >> 2);

	dim3 grid2((threads + 64 - 1) / 64);
	dim3 block2(64);

	dim3 grid3((threads + tpb3 - 1) / tpb3);
	dim3 block3(tpb3);
	
	if (device_sm[dev_id] >= 500)
	{
		lyra2h_gpu_hash_32_1 <<< grid2, block2 >>> (threads, startNounce, (uint2*)d_hash);

		lyra2h_gpu_hash_32_2 <<< grid1, block1, 768 *sizeof(uint2) * tpb >>> (threads, startNounce, d_hash);

		lyra2h_gpu_hash_32_3 <<< grid2, block2 >>> (threads, startNounce, (uint2*)d_hash, d_GNonces[thr_id]);
	}
	else if (device_sm[dev_id] == 350 || device_sm[dev_id] == 370)
	{

		uint32_t tpb35 = 16;
		dim3 grid35_ws(threads / tpb35);
		dim3 block35_ws(4 * tpb35);

		lyra2h_gpu_hash_32_ws2 << <grid35_ws, block35_ws >> > (threads, startNounce, (uint2*)d_hash, d_GNonces[thr_id]);

	}
	else 
		lyra2h_gpu_hash_32_sm2 <<< grid3, block3 >>> (threads, startNounce, d_hash, d_GNonces[thr_id]);



	// get first found nonce
	cudaMemcpy(h_GNonces[thr_id], d_GNonces[thr_id], 1 * sizeof(uint32_t), cudaMemcpyDeviceToHost);
	result = *h_GNonces[thr_id];

	return result;

}


<|start_filename|>lyra2/cuda_lyra2Z_sm5.cuh<|end_filename|>
#include <memory.h>

#ifdef __INTELLISENSE__
/* just for vstudio code colors */
#endif

#include "cuda_helper.h"
#include "cuda_lyra2_vectors.h"
#if __CUDA_ARCH__ == 350 || __CUDA_ARCH__ == 370
#define memshift 4
#else
#define memshift 3
#endif
#if __CUDA_ARCH__ < 500
//#define __ldg4t(x) (*x)  // prevent some issue when global mem isn't sync'd with its cache (but definetely slower)
#define vectype ulonglong4
#define u64type uint64_t
#define memshift 4

#elif __CUDA_ARCH__ == 500
#define vectype ulonglong4
#define u64type uint64_t
#define memshift 3 

#else  
#define vectype uint28
#define u64type uint2
#define memshift 3   

#endif 
#define vectype uint28
#define vectype uint28
#define u64type uint2
//#define memshift 3   
//#define __ldg4t(x) (*x)

__device__  uint2 *DMatrix;
__device__ vectype *DMatrix35;
#define TPB50 32
static __device__ __forceinline__ void Gfunc(uint2 & a, uint2 &b, uint2 &c, uint2 &d)
{
	a += b; d ^= a; d = SWAPUINT2(d);
	c += d; b ^= c; b = ROR2(b, 24);
	a += b; d ^= a; d = ROR2(d, 16);
	c += d; b ^= c; b = ROR2(b, 63);
}

static __device__ __forceinline__ void Gfunc(uint64_t & a, uint64_t &b, uint64_t &c, uint64_t &d)
{

	a += b; d ^= a; d = ROTR64(d, 32);
	c += d; b ^= c; b = ROTR64(b, 24);
	a += b; d ^= a; d = ROTR64(d, 16);
	c += d; b ^= c; b = ROTR64(b, 63);

}

static __device__ __forceinline__ void Gfunc_v35(uint2 & a, uint2 &b, uint2 &c, uint2 &d)
{
	/*
	a += b; d ^= a; d = ROR2(d, 32);
	c += d; b ^= c; b = ROR2(b, 24);
	a += b; d ^= a; d = ROR2(d, 16);
	c += d; b ^= c; b = ROR2(b, 63);
	*/

	a += b; d ^= a; d = SWAPUINT2(d);
	c += d; b ^= c; b = ROR24(b);
	a += b; d ^= a; d = ROR16(d);
	c += d; b ^= c; b = ROR2(b, 63);

}

static __device__ __forceinline__ void Gfunc_v35(unsigned long long & a, unsigned long long &b, unsigned long long &c, unsigned long long &d)
{

	a += b; d ^= a; d = ROTR64(d, 32);
	c += d; b ^= c; b = ROTR64(b, 24);
	a += b; d ^= a; d = ROTR64(d, 16);
	c += d; b ^= c; b = ROTR64(b, 63);

}

static __device__ __forceinline__ void round_lyra_v35(uint28* s)
{

	Gfunc_v35(s[0].x, s[1].x, s[2].x, s[3].x);
	Gfunc_v35(s[0].y, s[1].y, s[2].y, s[3].y);
	Gfunc_v35(s[0].z, s[1].z, s[2].z, s[3].z);
	Gfunc_v35(s[0].w, s[1].w, s[2].w, s[3].w);
	Gfunc_v35(s[0].x, s[1].y, s[2].z, s[3].w);
	Gfunc_v35(s[0].y, s[1].z, s[2].w, s[3].x);
	Gfunc_v35(s[0].z, s[1].w, s[2].x, s[3].y);
	Gfunc_v35(s[0].w, s[1].x, s[2].y, s[3].z);
}

static __device__ __forceinline__ void round_lyra_v35(ulonglong4* s)
{

	Gfunc_v35(s[0].x, s[1].x, s[2].x, s[3].x);
	Gfunc_v35(s[0].y, s[1].y, s[2].y, s[3].y);
	Gfunc_v35(s[0].z, s[1].z, s[2].z, s[3].z);
	Gfunc_v35(s[0].w, s[1].w, s[2].w, s[3].w);

	Gfunc_v35(s[0].x, s[1].y, s[2].z, s[3].w);
	Gfunc_v35(s[0].y, s[1].z, s[2].w, s[3].x);
	Gfunc_v35(s[0].z, s[1].w, s[2].x, s[3].y);
	Gfunc_v35(s[0].w, s[1].x, s[2].y, s[3].z);

}

static __device__ __forceinline__ void round_lyra_v35_ws(vectype &s)
{

	Gfunc_v35(s.x, s.y, s.z, s.w);

	s.y = shuffle2(s.y, threadIdx.x % 4 + 1, 4); //rotate
	s.z = shuffle2(s.z, threadIdx.x % 4 + 2, 4);
	s.w = shuffle2(s.w, threadIdx.x % 4 - 1, 4);

	Gfunc_v35(s.x, s.y, s.z, s.w);

	s.y = shuffle2(s.y, threadIdx.x % 4 - 1, 4); //rotate back
	s.z = shuffle2(s.z, threadIdx.x % 4 - 2, 4);
	s.w = shuffle2(s.w, threadIdx.x % 4 + 1, 4);


}

//#if __CUDA_ARCH__ == 500 || __CUDA_ARCH__ == 350


#define Nrow 8
#define Ncol 8
//#define memshift 3

#define BUF_COUNT 0



__device__ __forceinline__ uint32_t WarpShuffle(uint32_t a, uint32_t b, uint32_t c)
{
#if __CUDA_ARCH__ >= 300
	return __shfl(a, b, c);
#else
	extern __shared__ uint2 shared_mem[];

	const uint32_t thread = blockDim.x * threadIdx.y + threadIdx.x;
	uint32_t *_ptr = (uint32_t*)shared_mem;

	__threadfence_block();
	uint32_t buf = _ptr[thread];

	_ptr[thread] = a;
	__threadfence_block();
	uint32_t result = _ptr[(thread&~(c - 1)) + (b&(c - 1))];

	__threadfence_block();
	_ptr[thread] = buf;

	__threadfence_block();
	return result;
#endif
}

__device__ __forceinline__ uint2 WarpShuffle(uint2 a, uint32_t b, uint32_t c)
{
#if __CUDA_ARCH__ >= 300
	return make_uint2(__shfl(a.x, b, c), __shfl(a.y, b, c));
#else
	extern __shared__ uint2 shared_mem[];

	const uint32_t thread = blockDim.x * threadIdx.y + threadIdx.x;

	__threadfence_block();
	uint2 buf = shared_mem[thread];

	shared_mem[thread] = a;
	__threadfence_block();
	uint2 result = shared_mem[(thread&~(c - 1)) + (b&(c - 1))];

	__threadfence_block();
	shared_mem[thread] = buf;

	__threadfence_block();
	return result;
#endif
}

__device__ __forceinline__ void WarpShuffle3(uint2 &a1, uint2 &a2, uint2 &a3, uint32_t b1, uint32_t b2, uint32_t b3, uint32_t c)
{
#if __CUDA_ARCH__ >= 300
	a1 = WarpShuffle(a1, b1, c);
	a2 = WarpShuffle(a2, b2, c);
	a3 = WarpShuffle(a3, b3, c);
#else

	extern __shared__ uint2 shared_mem[];

	const uint32_t thread = blockDim.x * threadIdx.y + threadIdx.x;

	__threadfence_block();
	uint2 buf = shared_mem[thread];

	shared_mem[thread] = a1;
	__threadfence_block();
	a1 = shared_mem[(thread&~(c - 1)) + (b1&(c - 1))];
	__threadfence_block();
	shared_mem[thread] = a2;
	__threadfence_block();
	a2 = shared_mem[(thread&~(c - 1)) + (b2&(c - 1))];
	__threadfence_block();
	shared_mem[thread] = a3;
	__threadfence_block();
	a3 = shared_mem[(thread&~(c - 1)) + (b3&(c - 1))];

	__threadfence_block();
	shared_mem[thread] = buf;
	__threadfence_block();
#endif
}


__device__ __forceinline__ void ST4S(const int row, const int col, const uint2 data[3], const int thread, const int threads)
{

	extern __shared__ uint2 shared_mem[];
	const int s0 = (Ncol * row  + col) * memshift;
#pragma unroll
	for (int j = 0; j < 3; j++)
		shared_mem[((s0 + j) * blockDim.y + threadIdx.y) * blockDim.x + threadIdx.x] = data[j];

}

__device__ __forceinline__ void LD4S2(uint2 res[3], const int row, const int col, const int thread, const int threads, const uint2 * __restrict__ shared_mem)
{

//	extern __shared__ uint2 shared_mem[];
	const int s0 = (Ncol * row  + col) * memshift;

#pragma unroll
	for (int j = 0; j < 3; j++)
		res[j] = shared_mem[((s0 + j) * blockDim.y + threadIdx.y) * blockDim.x + threadIdx.x];

}

__device__ __forceinline__ void ST4S2(const int row, const int col, const uint2 data[3], const int thread, const int threads, uint2 * __restrict__ shared_mem)
{

//	extern __shared__ uint2 shared_mem[];
	const int s0 = (Ncol * row + col) * memshift;
#pragma unroll
	for (int j = 0; j < 3; j++)
		shared_mem[((s0 + j) * blockDim.y + threadIdx.y) * blockDim.x + threadIdx.x] = data[j];

}
__device__ __forceinline__ void ST4S2bis(const int row, const int col, const uint2 data[3], const int thread, const int threads, uint2 * __restrict__ shared_mem)
{

	//	extern __shared__ uint2 shared_mem[];
	const int s0 = (Ncol * row + col) * memshift;
#pragma unroll
	for (int j = 0; j < 3; j++)
		shared_mem[((s0 + j) * blockDim.y + threadIdx.y) * blockDim.x + threadIdx.x] = data[j];

}
__device__ __forceinline__ void LD4S(uint2 res[3], const int row, const int col, const int thread, const int threads)
{

	extern __shared__ uint2 shared_mem[];
	const int s0 = (Ncol * row + col) * memshift;

#pragma unroll
	for (int j = 0; j < 3; j++)
		res[j] = shared_mem[((s0 + j) * blockDim.y + threadIdx.y) * blockDim.x + threadIdx.x];

}



__device__ __forceinline__ uint2 LD4S(const int index)
{
	extern __shared__ uint2 shared_mem[];

	return shared_mem[(index * blockDim.y + threadIdx.y) * blockDim.x + threadIdx.x];
}

__device__ __forceinline__ void ST4S(const int index, const uint2 data)
{
	extern __shared__ uint2 shared_mem[];

	shared_mem[(index * blockDim.y + threadIdx.y) * blockDim.x + threadIdx.x] = data;
}



__device__ __forceinline__ void round_lyra(uint2 s[4])
{
	Gfunc(s[0], s[1], s[2], s[3]);
	WarpShuffle3(s[1], s[2], s[3], threadIdx.x + 1, threadIdx.x + 2, threadIdx.x + 3, 4);
	Gfunc(s[0], s[1], s[2], s[3]);
	WarpShuffle3(s[1], s[2], s[3], threadIdx.x + 3, threadIdx.x + 2, threadIdx.x + 1, 4);
}

static __device__ __forceinline__
void round_lyra(uint2x4* s)
{
	Gfunc(s[0].x, s[1].x, s[2].x, s[3].x);
	Gfunc(s[0].y, s[1].y, s[2].y, s[3].y);
	Gfunc(s[0].z, s[1].z, s[2].z, s[3].z);
	Gfunc(s[0].w, s[1].w, s[2].w, s[3].w);
	Gfunc(s[0].x, s[1].y, s[2].z, s[3].w);
	Gfunc(s[0].y, s[1].z, s[2].w, s[3].x);
	Gfunc(s[0].z, s[1].w, s[2].x, s[3].y);
	Gfunc(s[0].w, s[1].x, s[2].y, s[3].z);
}

static __device__ __forceinline__
void reduceDuplex(uint2 state[4], uint32_t thread, const uint32_t threads, uint2 * __restrict__ shared_mem)
{
	uint2 state1[3];

#if __CUDA_ARCH__ > 500
#pragma unroll
#endif
	for (int i = 0; i < Nrow; i++)
	{
		ST4S2(0, Ncol - i - 1, state, thread, threads,shared_mem);

		round_lyra(state);
	}

#pragma unroll 
	for (int i = 0; i < Nrow; i++)
	{
		LD4S2(state1, 0, i, thread, threads, shared_mem);
		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j];

		round_lyra(state);

		for (int j = 0; j < 3; j++)
			state1[j] ^= state[j];
		ST4S2(1, Ncol - i - 1, state1, thread, threads, shared_mem);
	}
}

static __device__ __forceinline__
void reduceDuplexRowSetup(const int rowIn, const int rowInOut, const int rowOut, uint2 state[4], uint32_t thread, const uint32_t threads, uint2 * __restrict__ shared_mem)
{
	uint2 state1[3], state2[3];

#pragma unroll 1
	for (int i = 0; i < Nrow; i++)
	{
		LD4S2(state1, rowIn, i, thread, threads, shared_mem);
		LD4S2(state2, rowInOut, i, thread, threads, shared_mem);
		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j] + state2[j];

		round_lyra(state);

#pragma unroll
		for (int j = 0; j < 3; j++)
			state1[j] ^= state[j];

		ST4S2(rowOut, Ncol - i - 1, state1, thread, threads, shared_mem);

		//一個手前のスレッドからデータを貰う(同時に一個先のスレッドにデータを送る)
		uint2 Data0 = state[0];
		uint2 Data1 = state[1];
		uint2 Data2 = state[2];
		WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

		if (threadIdx.x == 0)
		{
			state2[0] ^= Data2;
			state2[1] ^= Data0;
			state2[2] ^= Data1;
		}
		else {
			state2[0] ^= Data0;
			state2[1] ^= Data1;
			state2[2] ^= Data2;
		}

		ST4S2(rowInOut, i, state2, thread, threads, shared_mem);
	}
}

static __device__ __forceinline__
void reduceDuplexRowt(const int rowIn, const int rowInOut, const int rowOut, uint2 state[4], const uint32_t thread, const uint32_t threads, uint2 * __restrict__ shared_mem)
{
	for (int i = 0; i < Nrow; i++)
	{
		uint2 state1[3], state2[3];

		LD4S2(state1, rowIn, i, thread, threads, shared_mem);
		LD4S2(state2, rowInOut, i, thread, threads, shared_mem);

#pragma unroll
		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j] + state2[j];

		round_lyra(state);

		//一個手前のスレッドからデータを貰う(同時に一個先のスレッドにデータを送る)
		uint2 Data0 = state[0];
		uint2 Data1 = state[1];
		uint2 Data2 = state[2];
		WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

		if (threadIdx.x == 0)
		{
			state2[0] ^= Data2;
			state2[1] ^= Data0;
			state2[2] ^= Data1;
		}
		else
		{
			state2[0] ^= Data0;
			state2[1] ^= Data1;
			state2[2] ^= Data2;
		}

		ST4S2(rowInOut, i, state2, thread, threads, shared_mem);

		LD4S2(state1, rowOut, i, thread, threads, shared_mem);

#pragma unroll
		for (int j = 0; j < 3; j++)
			state1[j] ^= state[j];

		ST4S2(rowOut, i, state1, thread, threads, shared_mem);
	}
}


static __device__ __forceinline__
void reduceDuplexRowt_8(const int rowInOut, uint2* state, const uint32_t thread, const uint32_t threads, uint2 * __restrict__ shared_mem)
{
	uint2 state1[3], state2[3], last[3];

	LD4S2(state1, 2, 0, thread, threads, shared_mem);
	LD4S2(last, rowInOut, 0, thread, threads, shared_mem);

#pragma unroll
	for (int j = 0; j < 3; j++)
		state[j] ^= state1[j] + last[j];

	round_lyra(state);

	//一個手前のスレッドからデータを貰う(同時に一個先のスレッドにデータを送る)
	uint2 Data0 = state[0];
	uint2 Data1 = state[1];
	uint2 Data2 = state[2];
	WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

	if (threadIdx.x == 0)
	{
		last[0] ^= Data2;
		last[1] ^= Data0;
		last[2] ^= Data1;
	}
	else {
		last[0] ^= Data0;
		last[1] ^= Data1;
		last[2] ^= Data2;
	}

	if (rowInOut == 5)
	{
#pragma unroll
		for (int j = 0; j < 3; j++)
			last[j] ^= state[j];
	}

	for (int i = 1; i < Nrow; i++)
	{
		LD4S2(state1, 2, i, thread, threads, shared_mem);
		LD4S2(state2, rowInOut, i, thread, threads, shared_mem);

#pragma unroll
		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j] + state2[j];

		round_lyra(state);
	}

#pragma unroll
	for (int j = 0; j < 3; j++)
		state[j] ^= last[j];
}


static __device__ __forceinline__
void reduceDuplexRowt_8_v2(const int rowIn, const int rowOut, const int rowInOut, uint2* state, const uint32_t thread, const uint32_t threads, uint2 * __restrict__ shared_mem)
{
	uint2 state1[3], state2[3], last[3];

	LD4S2(state1, rowIn, 0, thread, threads, shared_mem);
	LD4S2(last, rowInOut, 0, thread, threads, shared_mem);

#pragma unroll
	for (int j = 0; j < 3; j++)
		state[j] ^= state1[j] + last[j];

	round_lyra(state);

	//一個手前のスレッドからデータを貰う(同時に一個先のスレッドにデータを送る)
	uint2 Data0 = state[0];
	uint2 Data1 = state[1];
	uint2 Data2 = state[2];
	WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

	if (threadIdx.x == 0)
	{
		last[0] ^= Data2;
		last[1] ^= Data0;
		last[2] ^= Data1;
	}
	else {
		last[0] ^= Data0;
		last[1] ^= Data1;
		last[2] ^= Data2;
	}

	if (rowInOut == rowOut)
	{
#pragma unroll
		for (int j = 0; j < 3; j++)
			last[j] ^= state[j];
	}

	for (int i = 1; i < Nrow; i++)
	{
		LD4S2(state1, rowIn, i, thread, threads, shared_mem);
		LD4S2(state2, rowInOut, i, thread, threads, shared_mem);

#pragma unroll
		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j] + state2[j];

		round_lyra(state);
	}

#pragma unroll
	for (int j = 0; j < 3; j++)
		state[j] ^= last[j];
}



static __device__ __forceinline__
void reduceDuplexV5(uint2 state[4], const uint32_t thread, const uint32_t threads)
{
	uint2 state1[3], state2[3];

	const uint32_t ps0 = (memshift * Ncol * 0 * threads + thread)*blockDim.x + threadIdx.x;
	const uint32_t ps1 = (memshift * Ncol * 1 * threads + thread)*blockDim.x + threadIdx.x;
	const uint32_t ps2 = (memshift * Ncol * 2 * threads + thread)*blockDim.x + threadIdx.x;
	const uint32_t ps3 = (memshift * Ncol * 3 * threads + thread)*blockDim.x + threadIdx.x;
	const uint32_t ps4 = (memshift * Ncol * 4 * threads + thread)*blockDim.x + threadIdx.x;
	const uint32_t ps5 = (memshift * Ncol * 5 * threads + thread)*blockDim.x + threadIdx.x;
	const uint32_t ps6 = (memshift * Ncol * 6 * threads + thread)*blockDim.x + threadIdx.x;
	const uint32_t ps7 = (memshift * Ncol * 7 * threads + thread)*blockDim.x + threadIdx.x;

	for (int i = 0; i < 8; i++)
	{
		const uint32_t s0 = memshift * Ncol * 0 + (Ncol - 1 - i) * memshift;
		#pragma unroll
		for (int j = 0; j < 3; j++)
			ST4S(s0 + j, state[j]);
		round_lyra(state);
	}

	for (int i = 0; i < 8; i++)
	{
		const uint32_t s0 = memshift * Ncol * 0 + i * memshift;
		const uint32_t s1 = ps1 + (7 - i)*memshift* threads*blockDim.x;
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state1[j] = LD4S(s0 + j);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j];

		round_lyra(state);

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(uint2*)(DMatrix + s1 + j*threads*blockDim.x) = state1[j] ^ state[j];
	}

	// 1, 0, 2
	for (int i = 0; i < 8; i++)
	{
		const uint32_t s0 = memshift * Ncol * 0 + i * memshift;
		const uint32_t s1 = ps1 + i * memshift* threads*blockDim.x;
		const uint32_t s2 = ps2 + (7 - i)*memshift* threads*blockDim.x;
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state1[j] = *(uint2*)(DMatrix + s1 + j*threads*blockDim.x);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state2[j] = LD4S(s0 + j);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j] + state2[j];

		round_lyra(state);

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(uint2*)(DMatrix + s2 + j*threads*blockDim.x) = state1[j] ^ state[j];

		//一個手前のスレッドからデータを貰う(同時に一個先のスレッドにデータを送る)
		uint2 Data0 = state[0];
		uint2 Data1 = state[1];
		uint2 Data2 = state[2];
		WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

		if (threadIdx.x == 0)
		{
			state2[0] ^= Data2;
			state2[1] ^= Data0;
			state2[2] ^= Data1;
		}
		else
		{
			state2[0] ^= Data0;
			state2[1] ^= Data1;
			state2[2] ^= Data2;
		}

		#pragma unroll
		for (int j = 0; j < 3; j++)
			ST4S(s0 + j, state2[j]);
	}

	// 2, 1, 3
	for (int i = 0; i < 8; i++)
	{
		const uint32_t s1 = ps1 + i * memshift* threads*blockDim.x;
		const uint32_t s2 = ps2 + i * memshift* threads*blockDim.x;
		const uint32_t s3 = ps3 + (7 - i)*memshift* threads*blockDim.x;
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state1[j] = *(uint2*)(DMatrix + s2 + j*threads*blockDim.x);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state2[j] = *(uint2*)(DMatrix + s1 + j*threads*blockDim.x);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j] + state2[j];

		round_lyra(state);

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(uint2*)(DMatrix + s3 + j*threads*blockDim.x) = state1[j] ^ state[j];

		//一個手前のスレッドからデータを貰う(同時に一個先のスレッドにデータを送る)
		uint2 Data0 = state[0];
		uint2 Data1 = state[1];
		uint2 Data2 = state[2];
		WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

		if (threadIdx.x == 0)
		{
			state2[0] ^= Data2;
			state2[1] ^= Data0;
			state2[2] ^= Data1;
		} else  {
			state2[0] ^= Data0;
			state2[1] ^= Data1;
			state2[2] ^= Data2;
		}

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(uint2*)(DMatrix + s1 + j*threads*blockDim.x) = state2[j];
	}

	// 3, 0, 4
	for (int i = 0; i < 8; i++)
	{
		const uint32_t ls0 = memshift * Ncol * 0 + i * memshift;
		const uint32_t s0 = ps0 + i * memshift* threads*blockDim.x;
		const uint32_t s3 = ps3 + i * memshift* threads*blockDim.x;
		const uint32_t s4 = ps4 + (7 - i)*memshift* threads*blockDim.x;
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state1[j] = *(uint2*)(DMatrix + s3 + j*threads*blockDim.x);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state2[j] = LD4S(ls0 + j);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j] + state2[j];

		round_lyra(state);

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(uint2*)(DMatrix + s4 + j*threads*blockDim.x) = state1[j] ^ state[j];

		//一個手前のスレッドからデータを貰う(同時に一個先のスレッドにデータを送る)
		uint2 Data0 = state[0];
		uint2 Data1 = state[1];
		uint2 Data2 = state[2];
		WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

		if (threadIdx.x == 0)
		{
			state2[0] ^= Data2;
			state2[1] ^= Data0;
			state2[2] ^= Data1;
		} else {
			state2[0] ^= Data0;
			state2[1] ^= Data1;
			state2[2] ^= Data2;
		}

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(uint2*)(DMatrix + s0 + j*threads*blockDim.x) = state2[j];
	}

	// 4, 3, 5
	for (int i = 0; i < 8; i++)
	{
		const uint32_t s3 = ps3 + i * memshift* threads*blockDim.x;
		const uint32_t s4 = ps4 + i * memshift* threads*blockDim.x;
		const uint32_t s5 = ps5 + (7 - i)*memshift* threads*blockDim.x;
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state1[j] = *(uint2*)(DMatrix + s4 + j*threads*blockDim.x);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state2[j] = *(uint2*)(DMatrix + s3 + j*threads*blockDim.x);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j] + state2[j];

		round_lyra(state);

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(uint2*)(DMatrix + s5 + j*threads*blockDim.x) = state1[j] ^ state[j];

		//一個手前のスレッドからデータを貰う(同時に一個先のスレッドにデータを送る)
		uint2 Data0 = state[0];
		uint2 Data1 = state[1];
		uint2 Data2 = state[2];
		WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

		if (threadIdx.x == 0)
		{
			state2[0] ^= Data2;
			state2[1] ^= Data0;
			state2[2] ^= Data1;
		}
		else
		{
			state2[0] ^= Data0;
			state2[1] ^= Data1;
			state2[2] ^= Data2;
		}

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(uint2*)(DMatrix + s3 + j*threads*blockDim.x) = state2[j];
	}

	// 5, 2, 6
	for (int i = 0; i < 8; i++)
	{
		const uint32_t s2 = ps2 + i * memshift* threads*blockDim.x;
		const uint32_t s5 = ps5 + i * memshift* threads*blockDim.x;
		const uint32_t s6 = ps6 + (7 - i)*memshift* threads*blockDim.x;
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state1[j] = *(uint2*)(DMatrix + s5 + j*threads*blockDim.x);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state2[j] = *(uint2*)(DMatrix + s2 + j*threads*blockDim.x);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j] + state2[j];

		round_lyra(state);

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(uint2*)(DMatrix + s6 + j*threads*blockDim.x) = state1[j] ^ state[j];

		//一個手前のスレッドからデータを貰う(同時に一個先のスレッドにデータを送る)
		uint2 Data0 = state[0];
		uint2 Data1 = state[1];
		uint2 Data2 = state[2];
		WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

		if (threadIdx.x == 0)
		{
			state2[0] ^= Data2;
			state2[1] ^= Data0;
			state2[2] ^= Data1;
		}
		else
		{
			state2[0] ^= Data0;
			state2[1] ^= Data1;
			state2[2] ^= Data2;
		}

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(uint2*)(DMatrix + s2 + j*threads*blockDim.x) = state2[j];
	}

	// 6, 1, 7
	for (int i = 0; i < 8; i++)
	{
		const uint32_t s1 = ps1 + i * memshift* threads*blockDim.x;
		const uint32_t s6 = ps6 + i * memshift* threads*blockDim.x;
		const uint32_t s7 = ps7 + (7 - i)*memshift* threads*blockDim.x;
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state1[j] = *(uint2*)(DMatrix + s6 + j*threads*blockDim.x);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state2[j] = *(uint2*)(DMatrix + s1 + j*threads*blockDim.x);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j] + state2[j];

		round_lyra(state);

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(uint2*)(DMatrix + s7 + j*threads*blockDim.x) = state1[j] ^ state[j];

		//一個手前のスレッドからデータを貰う(同時に一個先のスレッドにデータを送る)
		uint2 Data0 = state[0];
		uint2 Data1 = state[1];
		uint2 Data2 = state[2];
		WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

		if (threadIdx.x == 0)
		{
			state2[0] ^= Data2;
			state2[1] ^= Data0;
			state2[2] ^= Data1;
		} else {
			state2[0] ^= Data0;
			state2[1] ^= Data1;
			state2[2] ^= Data2;
		}

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(uint2*)(DMatrix + s1 + j*threads*blockDim.x) = state2[j];
	}
}

static __device__ __forceinline__
void reduceDuplexRowV50(const int rowIn, const int rowInOut, const int rowOut, uint2 state[4], const uint32_t thread, const uint32_t threads)
{
	const uint32_t ps1 = (memshift * Ncol * rowIn	 * threads + thread) * blockDim.x + threadIdx.x;
	const uint32_t ps2 = (memshift * Ncol * rowInOut * threads + thread) * blockDim.x + threadIdx.x;
	const uint32_t ps3 = (memshift * Ncol * rowOut	 * threads + thread) * blockDim.x + threadIdx.x;

	#pragma unroll 1
	for (int i = 0; i < 8; i++)
	{
		uint2 state1[3], state2[3];

		const uint32_t s1 = ps1 + i*memshift*threads *blockDim.x;
		const uint32_t s2 = ps2 + i*memshift*threads *blockDim.x;
		const uint32_t s3 = ps3 + i*memshift*threads *blockDim.x;

		#pragma unroll
		for (int j = 0; j < 3; j++) {
			state1[j] = *(uint2*)(DMatrix + s1 + j*threads*blockDim.x);
			state2[j] = *(uint2*)(DMatrix + s2 + j*threads*blockDim.x);
		}

		#pragma unroll
		for (int j = 0; j < 3; j++) {
			state1[j] += state2[j];
			state[j] ^= state1[j];
		}

		round_lyra(state);

		//一個手前のスレッドからデータを貰う(同時に一個先のスレッドにデータを送る)
		uint2 Data0 = state[0];
		uint2 Data1 = state[1];
		uint2 Data2 = state[2];
		WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

		if (threadIdx.x == 0)
		{
			state2[0] ^= Data2;
			state2[1] ^= Data0;
			state2[2] ^= Data1;
		} else {
			state2[0] ^= Data0;
			state2[1] ^= Data1;
			state2[2] ^= Data2;
		}

		#pragma unroll
		for (int j = 0; j < 3; j++)
		{
			*(uint2*)(DMatrix + s2 + j*threads*blockDim.x) = state2[j];
			*(uint2*)(DMatrix + s3 + j*threads*blockDim.x) ^= state[j];
		}
	}
}

static __device__ __forceinline__
void reduceDuplexRowV50_8(const int rowInOut, uint2 state[4], const uint32_t thread, const uint32_t threads)
{
	const uint32_t ps1 = (memshift * Ncol * 2*threads + thread)*blockDim.x + threadIdx.x;
	const uint32_t ps2 = (memshift * Ncol * rowInOut *threads + thread)*blockDim.x + threadIdx.x;
	// const uint32_t ps3 = (memshift * Ncol * 5*threads + thread)*blockDim.x + threadIdx.x;

	uint2 state1[3], last[3];

	#pragma unroll
	for (int j = 0; j < 3; j++) {
		state1[j] = *(uint2*)(DMatrix + ps1 + j*threads*blockDim.x);
		last[j] = *(uint2*)(DMatrix + ps2 + j*threads*blockDim.x);
	}

	#pragma unroll
	for (int j = 0; j < 3; j++) {
		state1[j] += last[j];
		state[j] ^= state1[j];
	}

	round_lyra(state);

	//一個手前のスレッドからデータを貰う(同時に一個先のスレッドにデータを送る)
	uint2 Data0 = state[0];
	uint2 Data1 = state[1];
	uint2 Data2 = state[2];
	WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

	if (threadIdx.x == 0)
	{
		last[0] ^= Data2;
		last[1] ^= Data0;
		last[2] ^= Data1;
	} else {
		last[0] ^= Data0;
		last[1] ^= Data1;
		last[2] ^= Data2;
	}

	if (rowInOut == 5)
	{
		#pragma unroll
		for (int j = 0; j < 3; j++)
			last[j] ^= state[j];
	}

	for (int i = 1; i < 8; i++)
	{
		const uint32_t s1 = ps1 + i*memshift*threads *blockDim.x;
		const uint32_t s2 = ps2 + i*memshift*threads *blockDim.x;

		#pragma unroll
		for (int j = 0; j < 3; j++)
			state[j] ^= *(uint2*)(DMatrix + s1 + j*threads*blockDim.x) + *(uint2*)(DMatrix + s2 + j*threads*blockDim.x);

		round_lyra(state);
	}


#pragma unroll
	for (int j = 0; j < 3; j++)
		state[j] ^= last[j];

}


static __device__ __forceinline__
void reduceDuplexRowV50_8_v2(const int rowIn, const int rowOut,const int rowInOut, uint2 state[4], const uint32_t thread, const uint32_t threads)
{
	const uint32_t ps1 = (memshift * Ncol * rowIn * threads + thread)*blockDim.x + threadIdx.x;
	const uint32_t ps2 = (memshift * Ncol * rowInOut *threads + thread)*blockDim.x + threadIdx.x;
	// const uint32_t ps3 = (memshift * Ncol * 5*threads + thread)*blockDim.x + threadIdx.x;

	uint2 state1[3], last[3];

#pragma unroll
	for (int j = 0; j < 3; j++) {
		state1[j] = *(uint2*)(DMatrix + ps1 + j*threads*blockDim.x);
		last[j] = *(uint2*)(DMatrix + ps2 + j*threads*blockDim.x);
	}

#pragma unroll
	for (int j = 0; j < 3; j++) {
		state1[j] += last[j];
		state[j] ^= state1[j];
	}

	round_lyra(state);

	//一個手前のスレッドからデータを貰う(同時に一個先のスレッドにデータを送る)
	uint2 Data0 = state[0];
	uint2 Data1 = state[1];
	uint2 Data2 = state[2];
	WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

	if (threadIdx.x == 0)
	{
		last[0] ^= Data2;
		last[1] ^= Data0;
		last[2] ^= Data1;
	}
	else {
		last[0] ^= Data0;
		last[1] ^= Data1;
		last[2] ^= Data2;
	}

	if (rowInOut == rowOut)
	{
#pragma unroll
		for (int j = 0; j < 3; j++)
			last[j] ^= state[j];
	}

	for (int i = 1; i < 8; i++)
	{
		const uint32_t s1 = ps1 + i*memshift*threads *blockDim.x;
		const uint32_t s2 = ps2 + i*memshift*threads *blockDim.x;

#pragma unroll
		for (int j = 0; j < 3; j++)
			state[j] ^= *(uint2*)(DMatrix + s1 + j*threads*blockDim.x) + *(uint2*)(DMatrix + s2 + j*threads*blockDim.x);

		round_lyra(state);
	}


#pragma unroll
	for (int j = 0; j < 3; j++)
		state[j] ^= last[j];

}
////////////////////// sm5+ kernels ////////

__device__ __forceinline__ void reduceDuplexRowSetupV2_ws(const int rowIn, const int rowInOut, const int rowOut, vectype state[4], uint32_t thread, uint32_t thread2)
{


	vectype state2[3], state1[3];
	uint32_t ps1 = memshift * (rowIn + 8 * thread2);  // top bottom
	uint32_t ps2 = memshift * (rowInOut + 8 * thread2); //top bottom
	uint32_t ps3 = memshift * (rowOut + 8 * thread2);  //bottom top


#pragma unroll
	for (int j = 0; j < 3; j++)
		state1[j] = __ldg4(&((vectype*)(DMatrix35))[j + ps1]);
#pragma unroll
	for (int j = 0; j < 3; j++)
		state2[j] = __ldg4(&((vectype*)(DMatrix35))[j + ps2]);

	//		 if (thread == 0) printf("thread %d  the matrix content row0 %08llx row1 %08llx    \n", threadIdx.x % 8, state1[0].x, state2[0].x);

	vectype tmp[3];
#pragma unroll
	for (int j = 0; j < 3; j++) {
		tmp[j] = state1[j] + state2[j];
	}

	//////// accross the threads ///////
	//#pragma unroll 
	for (int i = 0; i<8; i++)
	{
		if (threadIdx.x % 8 == 0) {
			for (int j = 0; j < 3; j++)
				state[j] ^= tmp[j];

			round_lyra_v35(state);

		}
		if (i<7) {
			for (int j = 0; j<4; j++)
				state[j] = shuffle_up4(state[j], 1, 8);
			for (int j = 0; j < 3; j++)
				tmp[j] = shuffle_down4(tmp[j], 1, 8);
		}
	}


	//		if (thread == 0) printf("thread %d  the matrix content rowIn %08llx  state %08llx\n", threadIdx.x % 8, state1[0].x, state[0].x);
#pragma unroll 
	for (int j = 0; j<3; j++)
		state1[j] = shuffle_xor4(state1[j], 7, 8);

#pragma unroll 
	for (int j = 0; j < 3; j++)
		state1[j] ^= state[j];

#pragma unroll 
	for (int j = 0; j<3; j++)
		state[j] = shuffle_xor4(state[j], 7, 8);  //reverse order


#pragma unroll 
	for (int j = 0; j<3; j++)
		((vectype*)(DMatrix35))[j + ps3] = state1[j];

	//		if (thread == 0) printf("thread %d before last xor rowOut %08llx  rowInOut %08llx state %08llx\n", threadIdx.x % 8, state1[0].x, state2[0].x, state[0].x);

	((uint2*)state2)[0] ^= ((uint2*)state)[11];
	for (int j = 0; j < 11; j++)
		((uint2*)state2)[j + 1] ^= ((uint2*)state)[j];
#pragma unroll 
	for (int j = 0; j < 3; j++)
		((vectype*)(DMatrix35))[j + ps2] = state2[j];
#pragma unroll 
	for (int j = 0; j<3; j++)
		state[j] = shuffle_xor4(state[j], 7, 8); //reverse back (not entirely necessary)


												 //		if (thread == 0) printf("thread %d  the matrix content rowIn %08llx rowOut %08llx  rowInOut %08llx state %08llx\n", threadIdx.x % 8, (DMatrix35 + ps1)[0].x, state1[0].x, state2[0].x, state[0].x);

}

__device__ __forceinline__ void reduceDuplex_ws(vectype state[4], uint32_t thread, uint32_t thread2)
{


	vectype state1[3], tmp[3];
	uint32_t ps1 = memshift * (8 * thread2);
	uint32_t ps2 = memshift * (1 + 8 * thread2);

#pragma unroll
	for (int j = 0; j < 3; j++)
		state1[j] = __ldg4(&((vectype*)(DMatrix35))[j + ps1]);

#pragma unroll
	for (int j = 0; j < 3; j++)
		tmp[j] = state1[j];
#pragma unroll 
	for (int i = 0; i<8; i++)
	{

		if (threadIdx.x % 8 == 0) {
			for (int j = 0; j < 3; j++)
				state[j] ^= tmp[j];
			round_lyra_v35(state);

		}

		if (i<7) {
			for (int j = 0; j<4; j++)
				state[j] = shuffle_up4(state[j], 1, 8);

			for (int j = 0; j < 3; j++)
				tmp[j] = shuffle_down4(tmp[j], 1, 8);
		}

	}

	for (int j = 0; j<3; j++)
		state1[j] = shuffle_xor4(state1[j], 7, 8); // reverse order 

	for (int j = 0; j < 3; j++)
		state1[j] ^= state[j];


	for (int j = 0; j < 3; j++)
		((vectype*)(DMatrix35))[j + ps2] = state1[j]; //store // matrix end up in top->bottom order 
}

__device__ __forceinline__ void reduceDuplexRowtV2_ws(const int rowIn, int rowInOut, const int rowOut, vectype* state, uint32_t thread, uint32_t thread2)
{

	vectype state1[3], state2[3], tmp[3];
	//	if (thread == 0) printf("rowInOut %d\n",rowInOut);

	rowInOut = __shfl(rowInOut, 0, 8); //broadcast over the 8 lanes
									   //	if (thread == 0) printf("rowInOut %d\n", rowInOut);

	uint32_t ps1 = memshift *(rowIn + 8 * thread2);
	uint32_t ps2 = memshift *(rowInOut + 8 * thread2);
	uint32_t ps3 = memshift *(rowOut + 8 * thread2);

	uint32_t s1 = ps1;
	uint32_t s2 = ps2;
	uint32_t s3 = ps3;

#pragma unroll
	for (int j = 0; j < 3; j++)
		state1[j] = ((vectype*)(DMatrix35))[j + s1];

if( s1!=s2) {
#pragma unroll
	for (int j = 0; j < 3; j++)
		state2[j] = ((vectype*)(DMatrix35))[j + s2];
}
else {
	for (int j = 0; j < 3; j++)
		state2[j] = state1[j];
}

#pragma unroll
	for (int j = 0; j < 3; j++)
		state1[j] += state2[j];
#pragma unroll
	for (int j = 0; j < 3; j++)
		tmp[j] = state1[j];


	///////////////////////////////////////////////

	////////////////////////////////////////////////// 
	//#pragma unroll 

	for (int i = 0; i<8; i++)
	{

		if (threadIdx.x % 8 == 0) {
#pragma unroll
			for (int j = 0; j < 3; j++)
				state[j] ^= tmp[j];
			round_lyra_v35(state);
		}
		if (i<7) {
#pragma unroll
			for (int j = 0; j<4; j++)
				state[j] = shuffle_up4(state[j], 1, 8);
#pragma unroll
			for (int j = 0; j<3; j++)
				tmp[j] = shuffle_down4(tmp[j], 1, 8);
		}
	}

	/////////////////////////////////



	for (int j = 0; j<3; j++)
		state[j] = shuffle_xor4(state[j], 7, 8);  //reverse order


												  ///////////////////////////////////////////////////
	((uint2*)state2)[0] ^= ((uint2*)state)[11];
	for (int j = 0; j < 11; j++)
		((uint2*)state2)[j + 1] ^= ((uint2*)state)[j];

	if (rowInOut != rowOut) {

		for (int j = 0; j < 3; j++)
			((vectype*)(DMatrix35))[j + s2] = state2[j];

		for (int j = 0; j < 3; j++)
			((vectype*)(DMatrix35))[j + s3] ^= state[j];

	}
	else {

		for (int j = 0; j < 3; j++)
			state2[j] ^= state[j];

		for (int j = 0; j < 3; j++)
			((vectype*)(DMatrix35))[j + s2] = state2[j];
	}

	for (int j = 0; j<3; j++)
		state[j] = shuffle_xor4(state[j], 7, 8);  //reverse back



}


static __device__ __forceinline__ void reduceDuplexV3(ulonglong4 state[4], uint32_t thread)
{


	ulonglong4 state1[3];
	uint32_t ps1 = (256 * thread);
	//                     colomn             row
	uint32_t ps2 = (memshift * 7 * 8 + memshift * 1 + 64 * memshift * thread);

#pragma unroll 4
	for (int i = 0; i < 8; i++)
	{
		uint32_t s1 = ps1 + 8 * i *memshift;
		uint32_t s2 = ps2 - 8 * i *memshift;

		for (int j = 0; j < 3; j++)
			state1[j] = __ldg4(&((ulonglong4*)(DMatrix35))[j + s1]);

		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j];
		round_lyra_v35(state);

		for (int j = 0; j < 3; j++)
			state1[j] ^= state[j];


		for (int j = 0; j < 3; j++)
			((ulonglong4*)(DMatrix35))[j + s2] = state1[j];

	}

}


static __device__ __forceinline__ 
void reduceDuplexRowSetupV3(const int rowIn, const int rowInOut, const int rowOut, ulonglong4 state[4], uint32_t thread)
{


	ulonglong4 state2[3], state1[3];

	uint32_t ps1 = (memshift *  rowIn + 64 * memshift * thread);
	uint32_t ps2 = (memshift * rowInOut + 64 * memshift* thread);
	uint32_t ps3 = (8 * memshift * 7 + memshift *  rowOut + 64 * memshift * thread);
	/*
	uint32_t ps1 = (256 * thread);
	uint32_t ps2 = (256 * thread);
	uint32_t ps3 = (256 * thread);
	*/
#pragma nounroll 
	for (int i = 0; i < 8; i++)
	{
		uint32_t s1 = ps1 + 8 * i*memshift;
		uint32_t s2 = ps2 + 8 * i*memshift;
		uint32_t s3 = ps3 - 8 * i*memshift;

		for (int j = 0; j < 3; j++)
			state1[j] = __ldg4(&((ulonglong4*)(DMatrix35))[j + s1]);
if (s1!=s2)
		for (int j = 0; j < 3; j++)
			state2[j] = __ldg4(&((ulonglong4*)(DMatrix35))[j + s2]);
else 
		for (int j = 0; j < 3; j++)
			state2[j] = state1[j];

		for (int j = 0; j < 3; j++) {
			ulonglong4 tmp = state1[j] + state2[j];
			state[j] ^= tmp;
		}


		round_lyra_v35(state);

		for (int j = 0; j < 3; j++) {
			state1[j] ^= state[j];
			((ulonglong4*)(DMatrix35))[j + s3] = state1[j];
		}

		((uint2*)state2)[0] ^= ((uint2*)state)[11];
		for (int j = 0; j < 11; j++)
			((uint2*)state2)[j + 1] ^= ((uint2*)state)[j];



		for (int j = 0; j < 3; j++)
			((ulonglong4*)(DMatrix35))[j + s2] = state2[j];

	}


}

static __device__ __forceinline__ 
void reduceDuplexRowtV3(const int rowIn, const int rowInOut, const int rowOut, ulonglong4* state, uint32_t thread)
{

	ulonglong4 state1[3], state2[3];
	uint32_t ps1 = (memshift * rowIn + 64 * memshift * thread);
	uint32_t ps2 = (memshift * rowInOut + 64 * memshift * thread);
	uint32_t ps3 = (memshift * rowOut + 64 * memshift * thread);

#pragma nounroll 
	for (int i = 0; i < 8; i++)
	{
		uint32_t s1 = ps1 + 8 * i*memshift;
		uint32_t s2 = ps2 + 8 * i*memshift;
		uint32_t s3 = ps3 + 8 * i*memshift;


		for (int j = 0; j < 3; j++)
			state1[j] = __ldg4(&((ulonglong4*)(DMatrix35))[j + s1]);

//if (s1!=s2)
		for (int j = 0; j < 3; j++)
			state2[j] = __ldg4(&((ulonglong4*)(DMatrix35))[j + s2]);
/*else
		for (int j = 0; j < 3; j++)
			state2[j] = state1[j];
*/
		for (int j = 0; j < 3; j++)
			state1[j] += state2[j];

		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j];


		round_lyra_v35(state);

		((uint2*)state2)[0] ^= ((uint2*)state)[11];
		for (int j = 0; j < 11; j++)
			((uint2*)state2)[j + 1] ^= ((uint2*)state)[j];

		if (rowInOut != rowOut) {

			for (int j = 0; j < 3; j++)
				((ulonglong4*)(DMatrix35))[j + s2] = state2[j];

			for (int j = 0; j < 3; j++)
				((ulonglong4*)(DMatrix35))[j + s3] ^= state[j];

		} else {

			for (int j = 0; j < 3; j++)
				state2[j] ^= state[j];

			for (int j = 0; j < 3; j++)
				((ulonglong4*)(DMatrix35))[j + s2] = state2[j];
		}






	}
}

static __device__ __forceinline__ void reduceDuplex_ws2(vectype &state, uint32_t thread, vectype *__restrict__ tmp)
{


	vectype state1;

	uint32_t ps2 = (memshift * (Ncol - 1) + memshift * Ncol + Nrow * Ncol * memshift * thread);

#pragma unroll 4
	for (int i = 0; i < Ncol; i++)
	{
		uint32_t s2 = ps2 - i*memshift;

		state1 = tmp[i]; //tmp[i];
		state.x ^= state1.x;
		state.y ^= state1.y;
		state.z ^= state1.z;

		round_lyra_v35_ws(state);
		state1 ^= state;
		tmp[i] = state1;
		(DMatrix35 + s2)[threadIdx.x % 4] = state1;

	}

}

static __device__ __forceinline__ void reduceDuplexRowSetup_ws2_pass1(const int rowIn, const int rowInOut, const int rowOut, vectype &state, uint32_t thread, vectype *__restrict__ temp)
{

	uint32_t laneId = threadIdx.x & 3;
	vectype state2, state1;
	vectype rstate;

	uint32_t ps2 = (memshift * Ncol * rowInOut + Nrow * Ncol * memshift * thread);
	uint32_t ps3 = (memshift * (Ncol - 1) + memshift * Ncol * rowOut + Nrow * Ncol * memshift * thread);

	//#pragma unroll 1
	for (int i = 0; i < Ncol; i++)
	{

		uint32_t s2 = ps2 + i*memshift;
		uint32_t s3 = ps3 - i*memshift;


		state1 = temp[7 - i];
		state2 = __ldg4t(&(DMatrix35 + s2)[laneId]);

		vectype tmp = state1 + state2;
		state.x ^= tmp.x;
		state.y ^= tmp.y;
		state.z ^= tmp.z;



		round_lyra_v35_ws(state);


		state1 ^= state;
		(DMatrix35 + s3)[laneId] = state1;
		temp[7 - i] = state1;

		rstate.x = shuffle2(state.x, laneId - 1, 4);
		rstate.y = shuffle2(state.y, laneId - 1, 4);
		rstate.z = shuffle2(state.z, laneId - 1, 4);

		if (laneId == 0)
		{
			u64type tmp2 = rstate.z;
			rstate.z = rstate.y;
			rstate.y = rstate.x;
			rstate.x = tmp2;
		}
		state2 ^= rstate;
		(DMatrix35 + s2)[laneId] = state2;

	}


}

static __device__ __forceinline__ void reduceDuplexRowSetup_ws2_pass2(const int rowIn, const int rowInOut, const int rowOut, vectype &state, uint32_t thread, vectype *__restrict__ temp)
{


	vectype state2, state1;
	vectype rstate;

	uint32_t ps2 = (memshift * Ncol * rowInOut + Nrow * Ncol * memshift * thread);
	uint32_t ps3 = (memshift * (Ncol - 1) + memshift * Ncol * rowOut + Nrow * Ncol * memshift * thread);
	uint32_t laneId = threadIdx.x & 3;
	//#pragma unroll 1
	for (int i = 0; i < Ncol; i++)
	{

		uint32_t s2 = ps2 + i*memshift;
		uint32_t s3 = ps3 - i*memshift;

		state1 = temp[i];
		state2 = __ldg4t(&(DMatrix35 + s2)[laneId]);

		vectype tmp = state1 + state2;
		state.x ^= tmp.x;
		state.y ^= tmp.y;
		state.z ^= tmp.z;



		round_lyra_v35_ws(state);


		state1 ^= state;
		(DMatrix35 + s3)[laneId] = state1;
		temp[i] = state1;

		rstate.x = shuffle2(state.x, laneId - 1, 4);
		rstate.y = shuffle2(state.y, laneId - 1, 4);
		rstate.z = shuffle2(state.z, laneId - 1, 4);

		if (threadIdx.x % 4 == 0)
		{
			u64type tmp2 = rstate.z;
			rstate.z = rstate.y;
			rstate.y = rstate.x;
			rstate.x = tmp2;
		}
		state2 ^= rstate;
		(DMatrix35 + s2)[laneId] = state2;

	}


}

static __device__ __forceinline__ void reduceDuplexRow_ws2(const int rowIn, const int rowInOut, const int rowOut, vectype &state, uint32_t thread, vectype * __restrict__ temp)
{

	vectype state1, state2, rstate;
	uint32_t ps2 = (memshift * Ncol * rowInOut + Nrow * Ncol * memshift * thread);
	uint32_t ps3 = (memshift * Ncol * rowOut + Nrow * Ncol * memshift * thread);
	uint32_t laneId = threadIdx.x & 3;
	//#pragma unroll 1
	for (int i = 0; i < Ncol; i++)
	{

		uint32_t s2 = ps2 + i*memshift;
		uint32_t s3 = ps3 + i*memshift;

		state1 = temp[7 - i];

		if (rowIn != rowInOut)
			state2 = __ldg4t(&(DMatrix35 + s2)[laneId]);
		else {
			state2 = state1;

		}
		state1 += state2;


		state.x ^= state1.x;
		state.y ^= state1.y;
		state.z ^= state1.z;

		round_lyra_v35_ws(state);

		rstate.x = shuffle2(state.x, laneId - 1, 4);
		rstate.y = shuffle2(state.y, laneId - 1, 4);
		rstate.z = shuffle2(state.z, laneId - 1, 4);

		if (laneId == 0)
		{
			u64type tmp = rstate.z;
			rstate.z = rstate.y;
			rstate.y = rstate.x;
			rstate.x = tmp;
		}
		state2 ^= rstate;


		if (rowInOut != rowOut) {

			(DMatrix35 + s2)[laneId] = state2;
			temp[7 - i] = __ldg4t(&(DMatrix35 + s3)[laneId]);
			temp[7 - i] ^= state;
			(DMatrix35 + s3)[laneId] = temp[7 - i];

		}
		else {

			state2 ^= state;
			temp[7 - i] = state2;
			(DMatrix35 + s2)[laneId] = state2;
		}


	}
}

static __device__ __forceinline__ void reduceDuplexRow_ws2_v2(const int rowIn, const int rowInOut, const int rowOut, vectype &state, uint32_t thread, vectype * __restrict__ temp)
{

	vectype state1, state2, rstate;
	uint32_t ps2 = (memshift * Ncol * rowInOut + Nrow * Ncol * memshift * thread);
	uint32_t ps3 = (memshift * Ncol * rowOut + Nrow * Ncol * memshift * thread);
	uint32_t laneId = threadIdx.x & 3;
	//#pragma unroll 1
	for (int i = 0; i < Ncol; i++)
	{

		uint32_t s2 = ps2 + i*memshift;
		uint32_t s3 = ps3 + i*memshift;

		state1 = temp[7 - i];

		if (rowIn != rowInOut)
			state2 = __ldg4t(&(DMatrix35 + s2)[laneId]);
		else {
			state2 = state1;

		}
		state1 += state2;


		state.x ^= state1.x;
		state.y ^= state1.y;
		state.z ^= state1.z;

		round_lyra_v35_ws(state);

if (i==0) {

		rstate.x = shuffle2(state.x, laneId - 1, 4);
		rstate.y = shuffle2(state.y, laneId - 1, 4);
		rstate.z = shuffle2(state.z, laneId - 1, 4);

		if (laneId == 0)
		{
			u64type tmp = rstate.z;
			rstate.z = rstate.y;
			rstate.y = rstate.x;
			rstate.x = tmp;
		}
		state2 ^= rstate;


		if (rowInOut != rowOut) {
			temp[7 - i] = state2;
/*
			(DMatrix35 + s2)[laneId] = state2;
			temp[7 - i] = __ldg4t(&(DMatrix35 + s3)[laneId]);
			temp[7 - i] ^= state;
			(DMatrix35 + s3)[laneId] = temp[7 - i];
*/
		}
		else {

			state2 ^= state;
			temp[7 - i] = state2;
//			(DMatrix35 + s2)[laneId] = state2;
		}

	}
	}
//	temp[0] = __ldg4t(&(DMatrix35 + ps2)[laneId]);
}


<|start_filename|>lyra2/cuda_lyra2Z.cu<|end_filename|>
/**
 * Lyra2 (v1) cuda implementation based on djm34 work
 * tpruvot@github 2015, Nanashi 08/2016 (from 1.8-r2)
 * Lyra2Z implentation for Zcoin based on all the previous
 * djm34 2017
 **/

#include <stdio.h>
#include <memory.h>

#define TPB52 32
#define TPB30 160
#define TPB20 160


//#include "cuda_lyra2Z_sm2.cuh"
#include "cuda_lyra2Z_sm5.cuh"

#ifdef __INTELLISENSE__
/* just for vstudio code colors */
#endif


static uint32_t *h_GNonces[16]; // this need to get fixed as the rest of that routine
static uint32_t *d_GNonces[16];
__constant__ uint32_t pTarget[8];


__constant__ static uint2 blake2b_IV_sm2[8] = {
	{ 0xf3bcc908, 0x6a09e667 },
	{ 0x84caa73b, 0xbb67ae85 },
	{ 0xfe94f82b, 0x3c6ef372 },
	{ 0x5f1d36f1, 0xa54ff53a },
	{ 0xade682d1, 0x510e527f },
	{ 0x2b3e6c1f, 0x9b05688c },
	{ 0xfb41bd6b, 0x1f83d9ab },
	{ 0x137e2179, 0x5be0cd19 }
};

#define reduceDuplexRow(rowIn, rowInOut, rowOut) { \
	for (int i = 0; i < 8; i++) { \
		for (int j = 0; j < 12; j++) \
			state[j] ^= Matrix[12 * i + j][rowIn] + Matrix[12 * i + j][rowInOut]; \
		round_lyra_sm2(state); \
		for (int j = 0; j < 12; j++) \
			Matrix[j + 12 * i][rowOut] ^= state[j]; \
		Matrix[0 + 12 * i][rowInOut] ^= state[11]; \
		Matrix[1 + 12 * i][rowInOut] ^= state[0]; \
		Matrix[2 + 12 * i][rowInOut] ^= state[1]; \
		Matrix[3 + 12 * i][rowInOut] ^= state[2]; \
		Matrix[4 + 12 * i][rowInOut] ^= state[3]; \
		Matrix[5 + 12 * i][rowInOut] ^= state[4]; \
		Matrix[6 + 12 * i][rowInOut] ^= state[5]; \
		Matrix[7 + 12 * i][rowInOut] ^= state[6]; \
		Matrix[8 + 12 * i][rowInOut] ^= state[7]; \
		Matrix[9 + 12 * i][rowInOut] ^= state[8]; \
		Matrix[10+ 12 * i][rowInOut] ^= state[9]; \
		Matrix[11+ 12 * i][rowInOut] ^= state[10]; \
	} \
  }

#define absorbblock(in)  { \
	state[0] ^= Matrix[0][in]; \
	state[1] ^= Matrix[1][in]; \
	state[2] ^= Matrix[2][in]; \
	state[3] ^= Matrix[3][in]; \
	state[4] ^= Matrix[4][in]; \
	state[5] ^= Matrix[5][in]; \
	state[6] ^= Matrix[6][in]; \
	state[7] ^= Matrix[7][in]; \
	state[8] ^= Matrix[8][in]; \
	state[9] ^= Matrix[9][in]; \
	state[10] ^= Matrix[10][in]; \
	state[11] ^= Matrix[11][in]; \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
	round_lyra_sm2(state); \
  }


__device__ __forceinline__
static void round_lyra_sm2(uint2 *s)
{
	Gfunc(s[0], s[4], s[8], s[12]);
	Gfunc(s[1], s[5], s[9], s[13]);
	Gfunc(s[2], s[6], s[10], s[14]);
	Gfunc(s[3], s[7], s[11], s[15]);
	Gfunc(s[0], s[5], s[10], s[15]);
	Gfunc(s[1], s[6], s[11], s[12]);
	Gfunc(s[2], s[7], s[8], s[13]);
	Gfunc(s[3], s[4], s[9], s[14]);
}

__device__ __forceinline__
void reduceDuplexRowSetup(const int rowIn, const int rowInOut, const int rowOut, uint2 state[16], uint2 Matrix[96][8])
{
#if __CUDA_ARCH__ > 500
#pragma unroll
#endif
	for (int i = 0; i < 8; i++)
	{
#pragma unroll
		for (int j = 0; j < 12; j++)
			state[j] ^= Matrix[12 * i + j][rowIn] + Matrix[12 * i + j][rowInOut];

		round_lyra_sm2(state);

#pragma unroll
		for (int j = 0; j < 12; j++)
			Matrix[j + 84 - 12 * i][rowOut] = Matrix[12 * i + j][rowIn] ^ state[j];

		Matrix[0 + 12 * i][rowInOut] ^= state[11];
		Matrix[1 + 12 * i][rowInOut] ^= state[0];
		Matrix[2 + 12 * i][rowInOut] ^= state[1];
		Matrix[3 + 12 * i][rowInOut] ^= state[2];
		Matrix[4 + 12 * i][rowInOut] ^= state[3];
		Matrix[5 + 12 * i][rowInOut] ^= state[4];
		Matrix[6 + 12 * i][rowInOut] ^= state[5];
		Matrix[7 + 12 * i][rowInOut] ^= state[6];
		Matrix[8 + 12 * i][rowInOut] ^= state[7];
		Matrix[9 + 12 * i][rowInOut] ^= state[8];
		Matrix[10 + 12 * i][rowInOut] ^= state[9];
		Matrix[11 + 12 * i][rowInOut] ^= state[10];
	}
}


//// legacy kernel //////////////////
__global__ __launch_bounds__(TPB30, 1)
void lyra2Z_gpu_hash_32_sm2(uint32_t threads, uint32_t startNounce, uint64_t *g_hash, uint32_t *resNonces)
{
	uint32_t thread = (blockDim.x * blockIdx.x + threadIdx.x);
	const uint2 Mask[8] = {
		{ 0x00000020, 0x00000000 },{ 0x00000020, 0x00000000 },
		{ 0x00000020, 0x00000000 },{ 0x00000008, 0x00000000 },
		{ 0x00000008, 0x00000000 },{ 0x00000008, 0x00000000 },
		{ 0x00000080, 0x00000000 },{ 0x00000000, 0x01000000 }
	};
	if (thread < threads)
	{
		uint2 state[16];

#pragma unroll
		for (int i = 0; i<4; i++) {
			LOHI(state[i].x, state[i].y, g_hash[threads*i + thread]);
		} //password

#pragma unroll
		for (int i = 0; i<4; i++) {
			state[i + 4] = state[i];
		} //salt

#pragma unroll
		for (int i = 0; i<8; i++) {
			state[i + 8] = blake2b_IV_sm2[i];
		}

		// blake2blyra x2
		//#pragma unroll 24
		for (int i = 0; i<12; i++) {
			round_lyra_sm2(state);
		}

		for (int i = 0; i<8; i++)
			state[i] ^= Mask[i];


		for (int i = 0; i<12; i++) {
			round_lyra_sm2(state);
		}


		uint2 Matrix[96][8]; // not cool

							 // reducedSqueezeRow0
#pragma unroll 8
		for (int i = 0; i < 8; i++)
		{
#pragma unroll 12
			for (int j = 0; j<12; j++) {
				Matrix[j + 84 - 12 * i][0] = state[j];
			}
			round_lyra_sm2(state);
		}

		// reducedSqueezeRow1
#pragma unroll 8
		for (int i = 0; i < 8; i++)
		{
#pragma unroll 12
			for (int j = 0; j<12; j++) {
				state[j] ^= Matrix[j + 12 * i][0];
			}
			round_lyra_sm2(state);
#pragma unroll 12
			for (int j = 0; j<12; j++) {
				Matrix[j + 84 - 12 * i][1] = Matrix[j + 12 * i][0] ^ state[j];
			}
		}

		reduceDuplexRowSetup(1, 0, 2, state, Matrix);
		reduceDuplexRowSetup(2, 1, 3, state, Matrix);
		reduceDuplexRowSetup(3, 0, 4, state, Matrix);
		reduceDuplexRowSetup(4, 3, 5, state, Matrix);
		reduceDuplexRowSetup(5, 2, 6, state, Matrix);
		reduceDuplexRowSetup(6, 1, 7, state, Matrix);

		uint32_t rowa;
		uint32_t prev = 7;
		uint32_t iterator = 0;
		for (uint32_t i = 0; i<8; i++) {
			rowa = state[0].x & 7;
			reduceDuplexRow(prev, rowa, iterator);
			prev = iterator;
			iterator = (iterator + 3) & 7;
		}
		for (uint32_t i = 0; i<8; i++) {
			rowa = state[0].x & 7;
			reduceDuplexRow(prev, rowa, iterator);
			prev = iterator;
			iterator = (iterator - 1) & 7;
		}

		for (uint32_t i = 0; i<8; i++) {
			rowa = state[0].x & 7;
			reduceDuplexRow(prev, rowa, iterator);
			prev = iterator;
			iterator = (iterator + 3) & 7;
		}
		for (uint32_t i = 0; i<8; i++) {
			rowa = state[0].x & 7;
			reduceDuplexRow(prev, rowa, iterator);
			prev = iterator;
			iterator = (iterator - 1) & 7;
		}


		for (uint32_t i = 0; i<8; i++) {
			rowa = state[0].x & 7;
			reduceDuplexRow(prev, rowa, iterator);
			prev = iterator;
			iterator = (iterator + 3) & 7;
		}
		for (uint32_t i = 0; i<8; i++) {
			rowa = state[0].x & 7;
			reduceDuplexRow(prev, rowa, iterator);
			prev = iterator;
			iterator = (iterator - 1) & 7;
		}

		for (uint32_t i = 0; i<8; i++) {
			rowa = state[0].x & 7;
			reduceDuplexRow(prev, rowa, iterator);
			prev = iterator;
			iterator = (iterator + 3) & 7;
		}
		for (uint32_t i = 0; i<8; i++) {
			rowa = state[0].x & 7;
			reduceDuplexRow(prev, rowa, iterator);
			prev = iterator;
			iterator = (iterator - 1) & 7;
		}



		absorbblock(rowa);
		uint32_t nonce = startNounce + thread;
		if (((uint64_t*)state)[3] <= ((uint64_t*)pTarget)[3]) {
			atomicMin(&resNonces[1], resNonces[0]);
			atomicMin(&resNonces[0], nonce);
		}
	} //thread
}


////////////////////// sm5+ kernels ////////

__global__ __launch_bounds__(128, 1)
void lyra2Z_gpu_hash_32_1(uint32_t threads, uint32_t startNounce, uint2 *g_hash)
{
	const uint32_t thread = (blockDim.x * blockIdx.x + threadIdx.x);
	const uint2x4 Mask[2] = {
		0x00000020UL, 0x00000000UL, 0x00000020UL, 0x00000000UL,
		0x00000020UL, 0x00000000UL, 0x00000008UL, 0x00000000UL,
		0x00000008UL, 0x00000000UL, 0x00000008UL, 0x00000000UL,
		0x00000080UL, 0x00000000UL, 0x00000000UL, 0x01000000UL
	};
	const uint2x4 blake2b_IV[2] = {
		0xf3bcc908lu, 0x6a09e667lu,
		0x84caa73blu, 0xbb67ae85lu,
		0xfe94f82blu, 0x3c6ef372lu,
		0x5f1d36f1lu, 0xa54ff53alu,
		0xade682d1lu, 0x510e527flu,
		0x2b3e6c1flu, 0x9b05688clu,
		0xfb41bd6blu, 0x1f83d9ablu,
		0x137e2179lu, 0x5be0cd19lu
	};
	if (thread < threads)
	{
		uint2x4 state[4];

		state[0].x = state[1].x = __ldg(&g_hash[thread + threads * 0]);
		state[0].y = state[1].y = __ldg(&g_hash[thread + threads * 1]);
		state[0].z = state[1].z = __ldg(&g_hash[thread + threads * 2]);
		state[0].w = state[1].w = __ldg(&g_hash[thread + threads * 3]);
		state[2] = blake2b_IV[0];
		state[3] = blake2b_IV[1];

		for (int i = 0; i<12; i++)
			round_lyra(state); //because 12 is not enough

		state[0] ^= Mask[0];
		state[1] ^= Mask[1];


		for (int i = 0; i<12; i++)
			round_lyra(state); //because 12 is not enough

		((uint2x4*)DMatrix)[threads * 0 + thread] = state[0];
		((uint2x4*)DMatrix)[threads * 1 + thread] = state[1];
		((uint2x4*)DMatrix)[threads * 2 + thread] = state[2];
		((uint2x4*)DMatrix)[threads * 3 + thread] = state[3];
	}
}

__global__
__launch_bounds__(TPB52, 1)
void lyra2Z_gpu_hash_32_2(uint32_t threads, uint32_t startNounce, uint64_t *g_hash)
{
	const uint32_t thread = blockDim.y * blockIdx.x + threadIdx.y;

//	uint32_t thread = (blockDim.x * blockIdx.x + threadIdx.x);
//	uint32_t thread2 = thread >> 2;


	__shared__ uint2 shared[24*8*32];
	if (thread < threads)
	{
		uint2 state[4];
		state[0] = __ldg(&((uint2*)DMatrix)[(0 * threads + thread) * blockDim.x + threadIdx.x]);
		state[1] = __ldg(&((uint2*)DMatrix)[(1 * threads + thread) * blockDim.x + threadIdx.x]);
		state[2] = __ldg(&((uint2*)DMatrix)[(2 * threads + thread) * blockDim.x + threadIdx.x]);
		state[3] = __ldg(&((uint2*)DMatrix)[(3 * threads + thread) * blockDim.x + threadIdx.x]);

		reduceDuplex(state, thread, threads,shared);
		reduceDuplexRowSetup(1, 0, 2, state, thread, threads, shared);
		reduceDuplexRowSetup(2, 1, 3, state, thread, threads, shared);
		reduceDuplexRowSetup(3, 0, 4, state, thread, threads, shared);
		reduceDuplexRowSetup(4, 3, 5, state, thread, threads, shared);
		reduceDuplexRowSetup(5, 2, 6, state, thread, threads, shared);
		reduceDuplexRowSetup(6, 1, 7, state, thread, threads, shared);


		uint32_t rowa; // = WarpShuffle(state[0].x, 0, 4) & 7;

		uint32_t prev = 7;
		uint32_t iterator = 0;


		//for (uint32_t j=0;j<4;j++) {

		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowt(prev, rowa, iterator, state, thread, threads, shared);
			prev = iterator;
			iterator = (iterator + 3) & 7;
		}

		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowt(prev, rowa, iterator, state, thread, threads, shared);
			prev = iterator;
			iterator = (iterator - 1) & 7;
		}

		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowt(prev, rowa, iterator, state, thread, threads, shared);
			prev = iterator;
			iterator = (iterator + 3) & 7;
		}

		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowt(prev, rowa, iterator, state, thread, threads, shared);
			prev = iterator;
			iterator = (iterator - 1) & 7;
		}

		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowt(prev, rowa, iterator, state, thread, threads, shared);
			prev = iterator;
			iterator = (iterator + 3) & 7;
		}

		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowt(prev, rowa, iterator, state, thread, threads, shared);
			prev = iterator;
			iterator = (iterator - 1) & 7;
		}

		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowt(prev, rowa, iterator, state, thread, threads, shared);
			prev = iterator;
			iterator = (iterator + 3) & 7;
		}

		for (uint32_t i = 0; i<7; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowt(prev, rowa, iterator, state, thread, threads, shared);
			prev = iterator;
			iterator = (iterator - 1) & 7;
		}

		//}
		rowa = WarpShuffle(state[0].x, 0, 4) & 7;
		reduceDuplexRowt_8_v2(prev, iterator, rowa, state, thread, threads, shared);


		((uint2*)DMatrix)[(0 * threads + thread) * blockDim.x + threadIdx.x] = state[0];
		((uint2*)DMatrix)[(1 * threads + thread) * blockDim.x + threadIdx.x] = state[1];
		((uint2*)DMatrix)[(2 * threads + thread) * blockDim.x + threadIdx.x] = state[2];
		((uint2*)DMatrix)[(3 * threads + thread) * blockDim.x + threadIdx.x] = state[3];
	}
}

 
__global__
__launch_bounds__(TPB52, 1)
void lyra2Z_gpu_hash_32_2bis(uint32_t threads, uint32_t startNounce, uint64_t *g_hash)
{
//	const uint32_t thread = blockDim.y * blockIdx.x + threadIdx.y;

	const	uint32_t thread = (blockDim.x * blockIdx.x + threadIdx.x);
	const	uint32_t thread2 = thread >> 2;


	__shared__ uint2 shared[24 * 8 * 32];
	if (thread < threads)
	{
		uint2 state[4];
/*
		state[0] = __ldg(&((uint2*)DMatrix)[thread2 + (threadIdx.x % 4) * threads]);
		state[1] = __ldg(&((uint2*)DMatrix)[thread2 + (threadIdx.x % 4) * threads]);
		state[2] = __ldg(&((uint2*)DMatrix)[thread2 + (threadIdx.x % 4) * threads]);
		state[3] = __ldg(&((uint2*)DMatrix)[thread2 + (threadIdx.x % 4) * threads]);
*/
		state[0] = __ldg(&((uint2*)DMatrix)[(0 * threads + thread2) * 4 + threadIdx.x %4]);
		state[1] = __ldg(&((uint2*)DMatrix)[(1 * threads + thread2) * 4 + threadIdx.x %4]);
		state[2] = __ldg(&((uint2*)DMatrix)[(2 * threads + thread2) * 4 + threadIdx.x %4]);
		state[3] = __ldg(&((uint2*)DMatrix)[(3 * threads + thread2) * 4 + threadIdx.x %4]);

/*
		((uint2x4*)DMatrix)[threads * 0 + thread] = state[0];
		((uint2x4*)DMatrix)[threads * 1 + thread] = state[1];
		((uint2x4*)DMatrix)[threads * 2 + thread] = state[2];
		((uint2x4*)DMatrix)[threads * 3 + thread] = state[3];
*/
		reduceDuplex(state, thread, threads, shared);
		reduceDuplexRowSetup(1, 0, 2, state, thread, threads, shared);
		reduceDuplexRowSetup(2, 1, 3, state, thread, threads, shared);
		reduceDuplexRowSetup(3, 0, 4, state, thread, threads, shared);
		reduceDuplexRowSetup(4, 3, 5, state, thread, threads, shared);
		reduceDuplexRowSetup(5, 2, 6, state, thread, threads, shared);
		reduceDuplexRowSetup(6, 1, 7, state, thread, threads, shared);


		uint32_t rowa; // = WarpShuffle(state[0].x, 0, 4) & 7;

		uint32_t prev = 7;
		uint32_t iterator = 0;


		//for (uint32_t j=0;j<4;j++) {

		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowt(prev, rowa, iterator, state, thread, threads, shared);
			prev = iterator;
			iterator = (iterator + 3) & 7;
		}

		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowt(prev, rowa, iterator, state, thread, threads, shared);
			prev = iterator;
			iterator = (iterator - 1) & 7;
		}

		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowt(prev, rowa, iterator, state, thread, threads, shared);
			prev = iterator;
			iterator = (iterator + 3) & 7;
		}

		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowt(prev, rowa, iterator, state, thread, threads, shared);
			prev = iterator;
			iterator = (iterator - 1) & 7;
		}

		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowt(prev, rowa, iterator, state, thread, threads, shared);
			prev = iterator;
			iterator = (iterator + 3) & 7;
		}

		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowt(prev, rowa, iterator, state, thread, threads, shared);
			prev = iterator;
			iterator = (iterator - 1) & 7;
		}

		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowt(prev, rowa, iterator, state, thread, threads, shared);
			prev = iterator;
			iterator = (iterator + 3) & 7;
		}

		for (uint32_t i = 0; i<7; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowt(prev, rowa, iterator, state, thread, threads, shared);
			prev = iterator;
			iterator = (iterator - 1) & 7;
		}

		//}
		rowa = WarpShuffle(state[0].x, 0, 4) & 7;
		reduceDuplexRowt_8_v2(prev, iterator, rowa, state, thread, threads, shared);

/*
		((uint2*)DMatrix)[(0 * threads + thread) * blockDim.x + threadIdx.x] = state[0];
		((uint2*)DMatrix)[(1 * threads + thread) * blockDim.x + threadIdx.x] = state[1];
		((uint2*)DMatrix)[(2 * threads + thread) * blockDim.x + threadIdx.x] = state[2];
		((uint2*)DMatrix)[(3 * threads + thread) * blockDim.x + threadIdx.x] = state[3];
*/
		state[0] = __ldg(&((uint2*)DMatrix)[(0 * threads + thread2) * 4 + threadIdx.x % 4]);
		state[1] = __ldg(&((uint2*)DMatrix)[(1 * threads + thread2) * 4 + threadIdx.x % 4]);
		state[2] = __ldg(&((uint2*)DMatrix)[(2 * threads + thread2) * 4 + threadIdx.x % 4]);
		state[3] = __ldg(&((uint2*)DMatrix)[(3 * threads + thread2) * 4 + threadIdx.x % 4]);

	}
}



__global__ __launch_bounds__(128, 1)
void lyra2Z_gpu_hash_32_3(uint32_t threads, uint32_t startNounce, uint2 *g_hash, uint32_t *resNonces)
{
	const uint32_t thread = blockDim.x * blockIdx.x + threadIdx.x;

	uint28 state[4];

	if (thread < threads)
	{
		state[0] = __ldg4(&((uint2x4*)DMatrix)[threads * 0 + thread]);
		state[1] = __ldg4(&((uint2x4*)DMatrix)[threads * 1 + thread]);
		state[2] = __ldg4(&((uint2x4*)DMatrix)[threads * 2 + thread]);
		state[3] = __ldg4(&((uint2x4*)DMatrix)[threads * 3 + thread]);

		for (int i = 0; i < 12; i++)
			round_lyra(state);
		uint32_t nonce = startNounce + thread;
		if (((uint64_t*)state)[3] <= ((uint64_t*)pTarget)[3]) {
			atomicMin(&resNonces[1], resNonces[0]);
			atomicMin(&resNonces[0], nonce);
		}
		/*
		g_hash[thread + threads * 0] = state[0].x;
		g_hash[thread + threads * 1] = state[0].y;
		g_hash[thread + threads * 2] = state[0].z;
		g_hash[thread + threads * 3] = state[0].w;
		*/
	} //thread
}

//////////////////////  sm5/sm35 kernels  ///// 

__global__ __launch_bounds__(128, 1)
void lyra2Z_gpu_hash_32_1_sm5(uint32_t threads, uint32_t startNounce, uint2 *g_hash)
{
	const uint32_t thread = (blockDim.x * blockIdx.x + threadIdx.x);

	const uint2x4 blake2b_IV[2] = {
		{ { 0xf3bcc908, 0x6a09e667 },{ 0x84caa73b, 0xbb67ae85 },{ 0xfe94f82b, 0x3c6ef372 },{ 0x5f1d36f1, 0xa54ff53a } },
		{ { 0xade682d1, 0x510e527f },{ 0x2b3e6c1f, 0x9b05688c },{ 0xfb41bd6b, 0x1f83d9ab },{ 0x137e2179, 0x5be0cd19 } }
	};
	const uint2x4 Mask[2] = {
		0x00000020UL, 0x00000000UL, 0x00000020UL, 0x00000000UL,
		0x00000020UL, 0x00000000UL, 0x00000008UL, 0x00000000UL,
		0x00000008UL, 0x00000000UL, 0x00000008UL, 0x00000000UL,
		0x00000080UL, 0x00000000UL, 0x00000000UL, 0x01000000UL
	};
	if (thread < threads)
	{
		uint2x4 state[4];

		((uint2*)state)[0] = __ldg(&g_hash[thread]);
		((uint2*)state)[1] = __ldg(&g_hash[thread + threads]);
		((uint2*)state)[2] = __ldg(&g_hash[thread + threads * 2]);
		((uint2*)state)[3] = __ldg(&g_hash[thread + threads * 3]);

		state[1] = state[0];
		state[2] = blake2b_IV[0];
		state[3] = blake2b_IV[1];

		for (int i = 0; i < 12; i++)
			round_lyra(state); //because 12 is not enough

		state[0] ^= Mask[0];
		state[1] ^= Mask[1];

		for (int i = 0; i < 12; i++)
			round_lyra(state); //because 12 is not enough


		((uint2x4*)DMatrix)[0 * threads + thread] = state[0];
		((uint2x4*)DMatrix)[1 * threads + thread] = state[1];
		((uint2x4*)DMatrix)[2 * threads + thread] = state[2];
		((uint2x4*)DMatrix)[3 * threads + thread] = state[3];
	}
} 

__global__ __launch_bounds__(TPB50, 1)
void lyra2Z_gpu_hash_32_2_sm5(uint32_t threads, uint32_t startNounce, uint2 *g_hash)
{
	const uint32_t thread = (blockDim.y * blockIdx.x + threadIdx.y);

	if (thread < threads)
	{
		uint2 state[4];

		state[0] = __ldg(&((uint2*)DMatrix)[(0 * threads + thread)*blockDim.x + threadIdx.x]);
		state[1] = __ldg(&((uint2*)DMatrix)[(1 * threads + thread)*blockDim.x + threadIdx.x]);
		state[2] = __ldg(&((uint2*)DMatrix)[(2 * threads + thread)*blockDim.x + threadIdx.x]);
		state[3] = __ldg(&((uint2*)DMatrix)[(3 * threads + thread)*blockDim.x + threadIdx.x]);
 
		reduceDuplexV5(state, thread, threads);

		uint32_t rowa; // = WarpShuffle(state[0].x, 0, 4) & 7;
		uint32_t prev = 7;
		uint32_t iterator = 0;
#pragma unroll 1
		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowV50(prev, rowa, iterator, state, thread, threads);
			prev = iterator;
			iterator = (iterator + 3) & 7;
		}
#pragma unroll 1
		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowV50(prev, rowa, iterator, state, thread, threads);
			prev = iterator;
			iterator = (iterator - 1) & 7;
		}
#pragma unroll 1
		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowV50(prev, rowa, iterator, state, thread, threads);
			prev = iterator;
			iterator = (iterator + 3) & 7;
		}
#pragma unroll 1
		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowV50(prev, rowa, iterator, state, thread, threads);
			prev = iterator;
			iterator = (iterator - 1) & 7;
		}
#pragma unroll 1
		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowV50(prev, rowa, iterator, state, thread, threads);
			prev = iterator;
			iterator = (iterator + 3) & 7;
		}
#pragma unroll 1
		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowV50(prev, rowa, iterator, state, thread, threads);
			prev = iterator;
			iterator = (iterator - 1) & 7;
		}
#pragma unroll 1
		for (uint32_t i = 0; i<8; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowV50(prev, rowa, iterator, state, thread, threads);
			prev = iterator;
			iterator = (iterator + 3) & 7;
		}
#pragma unroll 1 
		for (uint32_t i = 0; i<7; i++) {
			rowa = WarpShuffle(state[0].x, 0, 4) & 7;
			reduceDuplexRowV50(prev, rowa, iterator, state, thread, threads);
			prev = iterator;
			iterator = (iterator - 1) & 7;
		}

		rowa = WarpShuffle(state[0].x, 0, 4) & 7;
		reduceDuplexRowV50_8_v2(prev, iterator, rowa, state, thread, threads);



		((uint2*)DMatrix)[(0 * threads + thread)*blockDim.x + threadIdx.x] = state[0];
		((uint2*)DMatrix)[(1 * threads + thread)*blockDim.x + threadIdx.x] = state[1];
		((uint2*)DMatrix)[(2 * threads + thread)*blockDim.x + threadIdx.x] = state[2];
		((uint2*)DMatrix)[(3 * threads + thread)*blockDim.x + threadIdx.x] = state[3];
	}
}

__global__ __launch_bounds__(128, 1)
void lyra2Z_gpu_hash_32_3_sm5(uint32_t threads, uint32_t startNounce, uint2 *g_hash, uint32_t *resNonces)
{
	const uint32_t thread = (blockDim.x * blockIdx.x + threadIdx.x);

	if (thread < threads)
	{
		uint2x4 state[4];

		state[0] = __ldg4(&((uint2x4*)DMatrix)[0 * threads + thread]);
		state[1] = __ldg4(&((uint2x4*)DMatrix)[1 * threads + thread]);
		state[2] = __ldg4(&((uint2x4*)DMatrix)[2 * threads + thread]);
		state[3] = __ldg4(&((uint2x4*)DMatrix)[3 * threads + thread]);

		for (int i = 0; i < 12; i++)
			round_lyra(state);


		uint32_t nonce = startNounce + thread;
		if (((uint64_t*)state)[3] <= ((uint64_t*)pTarget)[3]) {
			atomicMin(&resNonces[1], resNonces[0]);
			atomicMin(&resNonces[0], nonce);
		}

	}
}

/***********************************************/





__global__	__launch_bounds__(48, 1)
void lyra2_gpu_hash_32_v3(uint32_t threads, uint32_t startNounce, uint2 *outputHash, uint32_t *resNonces)
{
	uint32_t thread = (blockDim.x * blockIdx.x + threadIdx.x);

	ulonglong4 state[4];
	

	const ulonglong4 blake2b_IV[2] = {
		{ 0x6a09e667f3bcc908,
		0xbb67ae8584caa73b,
		0x3c6ef372fe94f82b,
		0xa54ff53a5f1d36f1 },
		{ 0x510e527fade682d1,
		0x9b05688c2b3e6c1f,
		0x1f83d9abfb41bd6b,
		0x5be0cd19137e2179 } };
	
	const ulonglong4 Mask[2] = {
		{ 0x20,
		 0x20,
		 0x20,
		 0x08},
		{ 0x08,
		 0x08,
		 0x80,
		 0x0100000000000000 }
	};

#if __CUDA_ARCH__ == 350 || __CUDA_ARCH__ == 370
	if (thread < threads)
#endif
	{

		((uint2*)state)[0] = __ldg(&outputHash[thread]);
		((uint2*)state)[1] = __ldg(&outputHash[thread + threads]);
		((uint2*)state)[2] = __ldg(&outputHash[thread + threads * 2]);
		((uint2*)state)[3] = __ldg(&outputHash[thread + threads * 3]);

		state[1] = state[0];

		state[2] = ((ulonglong4*)blake2b_IV)[0];
		state[3] = ((ulonglong4*)blake2b_IV)[1];

		for (int i = 0; i<12; i++)
			round_lyra_v35(state);  //because 12 is not enough

		state[0] ^= Mask[0];
		state[1] ^= Mask[1];

		for (int i = 0; i<12; i++)
			round_lyra_v35(state);  //because 12 is not enough


		uint32_t ps1 = (8 * memshift * 7 + 64 * memshift * thread);


		for (int i = 0; i < 8; i++)
		{
			uint32_t s1 = ps1 - 8 * memshift * i;
			for (int j = 0; j < 3; j++)
				((ulonglong4*)(DMatrix35))[j + s1] = (state)[j];

			round_lyra_v35(state);
		}


		reduceDuplexV3(state, thread);

		reduceDuplexRowSetupV3(1, 0, 2, state, thread);
		reduceDuplexRowSetupV3(2, 1, 3, state, thread);
		reduceDuplexRowSetupV3(3, 0, 4, state, thread);
		reduceDuplexRowSetupV3(4, 3, 5, state, thread);
		reduceDuplexRowSetupV3(5, 2, 6, state, thread);
		reduceDuplexRowSetupV3(6, 1, 7, state, thread);

		uint32_t rowa;
		uint32_t prev = 7;
		uint32_t iterator = 0;
	   for (int j=0;j<4;j++) {
		for (uint32_t i = 0; i<8; i++) {
			rowa = ((uint2*)state)[0].x & 7;
			reduceDuplexRowtV3(prev, rowa, iterator, state, thread);
			prev = iterator;
			iterator = (iterator + 3) & 7;
		}
		
		for (uint32_t i = 0; i<8; i++) {
		rowa = ((uint2*)state)[0].x & 7;
		reduceDuplexRowtV3(prev, rowa, iterator, state, thread);
		prev = iterator;
		iterator = (iterator - 1) & 7;
		}
		
		}

		uint32_t shift = (memshift * rowa + 64 * memshift * thread);

		for (int j = 0; j < 3; j++)
			state[j] ^= __ldg4(&((ulonglong4*)(DMatrix35))[j + shift]);

		for (int i = 0; i < 12; i++)
			round_lyra_v35(state);

		uint32_t nonce = startNounce + thread;
		if (((uint64_t*)state)[3] <= ((uint64_t*)pTarget)[3]) {
			atomicMin(&resNonces[1], resNonces[0]);
			atomicMin(&resNonces[0], nonce);
		}

	} //thread
}



__global__	__launch_bounds__(4 * 16, 1)
void lyra2v2_gpu_hash_32_ws2(uint32_t threads, uint32_t startNounce, uint2 *outputHash, uint32_t *resNonces)
{
#if __CUDA_ARCH__ > 300 

		uint32_t thread = (blockDim.x * blockIdx.x + threadIdx.x);
		uint32_t thread2 = thread >> 2;

		vectype state;
		vectype temp[8];

 
		const uint28 blake2b_IV[2] = {
			{ 0xf3bcc908, 0x6a09e667,
			  0x84caa73b, 0xbb67ae85,
			  0xfe94f82b, 0x3c6ef372,
			  0x5f1d36f1, 0xa54ff53a },
			{ 0xade682d1, 0x510e527f,
			  0x2b3e6c1f, 0x9b05688c,
			  0xfb41bd6b, 0x1f83d9ab,
			  0x137e2179, 0x5be0cd19 } };

		const uint28 padding[2] = {
			{ 0x20, 0x0,
			  0x20, 0x0,
			  0x20, 0x0,
			  0x08, 0x0 },
			{ 0x08, 0x0,
			  0x08, 0x0,
			  0x80, 0x0,
			  0x00, 0x01000000 } };

//		state.x = __ldg(&((u64type*)outputHash)[4 * thread2 + (threadIdx.x % 4)]);

		state.x = __ldg(&((u64type*)outputHash)[thread2 + (threadIdx.x % 4) * threads]);
		state.y = state.x;

		state.z = ((u64type*)blake2b_IV)[threadIdx.x % 4];
		state.w = ((u64type*)blake2b_IV)[4 + threadIdx.x % 4];


	for (int i = 0; i<12; i++)
		round_lyra_v35_ws(state);

		state.x ^= ((u64type*)padding)[threadIdx.x % 4];
		state.y ^= ((u64type*)padding)[4 + threadIdx.x % 4];


	for (int i = 0; i<12; i++)
		round_lyra_v35_ws(state);

	uint32_t ps1 = (memshift * (Ncol - 1) + Nrow * Ncol * memshift * thread2);

	for (int i = 0; i < Ncol; i++)
	{
		uint32_t s1 = ps1 - memshift * i;
		((vectype*)(DMatrix35))[(threadIdx.x % 4) + s1] = state;
		temp[(7 - i)] = state;
		round_lyra_v35_ws(state);
	}

		reduceDuplex_ws2(state, thread2, temp);
		reduceDuplexRowSetup_ws2_pass1(1, 0, 2, state, thread2, temp);
		reduceDuplexRowSetup_ws2_pass2(2, 1, 3, state, thread2, temp);
		reduceDuplexRowSetup_ws2_pass1(3, 0, 4, state, thread2, temp);
		reduceDuplexRowSetup_ws2_pass2(4, 3, 5, state, thread2, temp);
		reduceDuplexRowSetup_ws2_pass1(5, 2, 6, state, thread2, temp);
		reduceDuplexRowSetup_ws2_pass2(6, 1, 7, state, thread2, temp);




		uint32_t rowa;
		uint32_t prev = 7;
		uint32_t iterator = 0;
	for (int j=0;j<4;j++) {
		for (int i = 0; i < 8; i++)
		{
			rowa = shuffle2t(state.x, 0, 4) & 7;  
			reduceDuplexRow_ws2(prev, rowa, iterator, state, thread2, temp);
			prev = iterator;
			iterator = (iterator + 3) & 7;
		}
		for (int i = 0; i < 8; i++)
		{
			rowa = shuffle2t(state.x, 0, 4) & 7;
if (i==7 && j==3)
			reduceDuplexRow_ws2_v2(prev, rowa, iterator, state, thread2, temp);
else 
			reduceDuplexRow_ws2(prev, rowa, iterator, state, thread2, temp);

			prev = iterator;
			iterator = (iterator - 1) & 7;
		}
	}
		
		uint32_t shift = (memshift * Ncol * rowa + Nrow * Ncol * memshift * thread2);
		vectype tmp2 = temp[7];//__ldg4t(&((vectype*)(DMatrix35))[(threadIdx.x % 4) + shift]);
		state.x ^= tmp2.x;
		state.y ^= tmp2.y;
		state.z ^= tmp2.z;

		for (int i = 0; i < 12; i++)
			round_lyra_v35_ws(state);

//	((u64type*)outputHash)[4 * thread2 + (threadIdx.x % 4)] = state.x;

		if ((threadIdx.x % 4) == 3) {
			uint32_t nonce = startNounce + thread2;
			if ( devectorize(state.x) <= ((uint64_t*)pTarget)[3]) {
				atomicMin(&resNonces[1], resNonces[0]);
				atomicMin(&resNonces[0], nonce);
			}
		}
#endif
}




__host__
void lyra2Z_cpu_init(int thr_id, uint32_t threads, uint64_t *d_matrix)
{

	// just assign the device pointer allocated in main loop
	cudaMemcpyToSymbol(DMatrix, &d_matrix, sizeof(uint64_t*), 0, cudaMemcpyHostToDevice);
	cudaMalloc(&d_GNonces[thr_id], 2 * sizeof(uint32_t));
	cudaMallocHost(&h_GNonces[thr_id], 2 * sizeof(uint32_t));
}

__host__
void lyra2Z_cpu_init_sm35(int thr_id, uint32_t threads, uint64_t *d_matrix)
{

	// just assign the device pointer allocated in main loop
	cudaMemcpyToSymbol(DMatrix35, &d_matrix, sizeof(uint64_t*), 0, cudaMemcpyHostToDevice);
	cudaMalloc(&d_GNonces[thr_id], 2 * sizeof(uint32_t));
	cudaMallocHost(&h_GNonces[thr_id], 2 * sizeof(uint32_t));
}


__host__
void lyra2Z_cpu_init_sm2(int thr_id, uint32_t threads)
{

	// just assign the device pointer allocated in main loop
	cudaMalloc(&d_GNonces[thr_id], 2 * sizeof(uint32_t));
	cudaMallocHost(&h_GNonces[thr_id], 2 * sizeof(uint32_t));
}
  
__host__
uint32_t lyra2Z_getSecNonce(int thr_id, int num)
{
	uint32_t results[2];
	memset(results, 0xFF, sizeof(results));
	cudaMemcpy(results, d_GNonces[thr_id], sizeof(results), cudaMemcpyDeviceToHost);
	if (results[1] == results[0])
		return UINT32_MAX;
	return results[num];
}

__host__
void lyra2Z_setTarget(const void *pTargetIn)
{
	cudaMemcpyToSymbol(pTarget, pTargetIn, 32, 0, cudaMemcpyHostToDevice);
}


__host__
uint32_t lyra2Z_cpu_hash_32(int thr_id, uint32_t threads, uint32_t startNounce, uint64_t *d_hash, bool gtx750ti)
{
	uint32_t result = UINT32_MAX;
	cudaMemset(d_GNonces[thr_id], 0xff, 2 * sizeof(uint32_t));
	int dev_id = device_map[thr_id % MAX_GPUS];

	uint32_t tpb = TPB52;

	if (device_sm[dev_id] == 500) 
		tpb = TPB50;
	if (device_sm[dev_id] == 200) 
		tpb = TPB20; 

	dim3 grid1((threads * 4 + tpb - 1) / tpb);
	dim3 block1(4, tpb >> 2);


	dim3 grid2((threads + 128 - 1) / 128);
	dim3 block2(128);

	dim3 grid3((threads + tpb - 1) / tpb);
	dim3 block3(tpb); 

	if (device_sm[dev_id] >= 520)
	{
		lyra2Z_gpu_hash_32_1 <<< grid2, block2 >>> (threads, startNounce, (uint2*)d_hash);

		lyra2Z_gpu_hash_32_2 << < grid1, block1>> > (threads, startNounce, d_hash);
//		lyra2Z_gpu_hash_32_2 <<< grid1, block1, 24 * (8 - 0) * sizeof(uint2) * tpb>>> (threads, startNounce, d_hash);

		lyra2Z_gpu_hash_32_3 <<< grid2, block2 >>> (threads, startNounce, (uint2*)d_hash, d_GNonces[thr_id]);
	}

	else if (device_sm[dev_id] == 500) // || device_sm[dev_id] == 350)
	{
		size_t shared_mem = 0;
		
		if (gtx750ti)
			// 8Warpに調整のため、8192バイト確保する
			shared_mem = 8192;
		else
			// 10Warpに調整のため、6144バイト確保する
			shared_mem = 6144;

		lyra2Z_gpu_hash_32_1_sm5 <<< grid2, block2 >>> (threads, startNounce, (uint2*)d_hash);

		lyra2Z_gpu_hash_32_2_sm5 <<< grid1, block1, shared_mem >>> (threads, startNounce, (uint2*)d_hash);

		lyra2Z_gpu_hash_32_3_sm5 <<< grid2, block2 >>> (threads, startNounce, (uint2*)d_hash, d_GNonces[thr_id]);
	}

	else 
	if (device_sm[dev_id] == 350 || device_sm[dev_id] == 370)
	{ 

			uint32_t tpb35 = 16;
			dim3 grid35_ws(threads / tpb35);
			dim3 block35_ws(4 * tpb35);

			lyra2v2_gpu_hash_32_ws2 << <grid35_ws, block35_ws >> > (threads, startNounce, (uint2*)d_hash, d_GNonces[thr_id]);

	}
	else
		lyra2Z_gpu_hash_32_sm2 <<< grid3, block3 >>> (threads, startNounce, d_hash, d_GNonces[thr_id]);

	// get first found nonce
	cudaMemcpy(h_GNonces[thr_id], d_GNonces[thr_id], 1 * sizeof(uint32_t), cudaMemcpyDeviceToHost);
	result = *h_GNonces[thr_id];

	return result;

}
