I tried implementing AVX version. On my PC it runs 3 times faster
Duration symsum_avx: 0.127393 s
Duration symsum_forward_unoptimized: 0.370589 s
EDIT: changed loading and storing to unaligned versions as suggested
here’s the code, if you’re interested
torch::Tensor symsum_avx(torch::Tensor input)
{
	const torch::Tensor& output = torch::empty_like(input);

	const int batch_stride = input.strides()[0];
	const int half_batch_stride = batch_stride / 2;

	float* in_i1 = input.data_ptr<float>(); 
	float* in_i2 = in_i1 + half_batch_stride;
	float* out_i1 = output.data_ptr<float>();
	float* out_i2 = out_i1 + half_batch_stride;

	float* batch_end = in_i2 + half_batch_stride;
	const float* end = input.data_ptr<float>() + input.numel();

	const __m256 avx_zeros = _mm256_setzero_ps();

	float all_bits_on = 0.0f;
	memset(&all_bits_on, ~0, 4);
	const __m256 avx_all_bits_on = _mm256_set1_ps(all_bits_on);

	while (in_i2 < end)
	{
		while ((batch_end - in_i2) >= 8)
		{
			auto in1 = _mm256_loadu_ps(in_i1);
			auto in2 = _mm256_loadu_ps(in_i2);

			const auto b1 = _mm256_cmp_ps(in1, avx_zeros, _CMP_LT_OQ);
			const auto b2 = _mm256_cmp_ps(in2, avx_zeros, _CMP_LT_OQ);

			const auto c1 = _mm256_and_ps(b1, b2);
			const auto c2 = _mm256_andnot_ps(b1, b2);
			const auto c3 = _mm256_andnot_ps(b2, b1); 
			const auto c4 = _mm256_andnot_ps(b1, _mm256_andnot_ps(b2, avx_all_bits_on));

			const auto sum = _mm256_add_ps(in1, in2);

			auto out1 = _mm256_blendv_ps(avx_zeros, in1, c4);
			out1 = _mm256_blendv_ps(out1, sum, c2);
			out1 = _mm256_blendv_ps(out1, in2, c1);

			auto out2 = _mm256_blendv_ps(avx_zeros, in2, c4);
			out2 = _mm256_blendv_ps(out2, sum, c3);
			out2 = _mm256_blendv_ps(out2, in1, c1);

			_mm256_storeu_ps(out_i1, out1);
			_mm256_storeu_ps(out_i2, out2);
			out_i1 += 8; out_i2 += 8; in_i1 += 8; in_i2 += 8;
		}

		while (in_i2 < batch_end)
		{
			const bool b1 = std::signbit(*in_i1);
			const bool b2 = std::signbit(*in_i2);

			const bool c1 = b1 && b2;
			const bool c2 = !b1 && b2;
			const bool c3 = b1 && !b2;
			const bool c4 = !b1 && !b2;

			const float sum = (*in_i2) + (*in_i1);

			*out_i1++ = c1 ? *in_i2 : (c2 ? sum : (c4 ? *in_i1 : *out_i1));
			*out_i2++ = c1 ? *in_i1 : (c3 ? sum : (c4 ? *in_i2 : *out_i2));

			++in_i1;
			++in_i2;
		}

		in_i1 += half_batch_stride;
		in_i2 += half_batch_stride;
		out_i1 += half_batch_stride;
		out_i2 += half_batch_stride;
		batch_end += batch_stride;
	}

	return output;
}

How I tested:

void main()
{
	auto input = torch::randn({ 10000, 10000 });

	auto avx_start = std::chrono::steady_clock::now();
	auto output_avx = symsum_avx(input);
	auto avx_end = std::chrono::steady_clock::now();

	auto default_start = std::chrono::steady_clock::now();
	auto output_default = symsum_forward_unoptimized(input);
	auto default_end = std::chrono::steady_clock::now();

	std::cout << "Duration symsum_avx: " << (std::chrono::duration_cast<std::chrono::microseconds>(avx_end - avx_start).count() / 1000000.0) << " s" << std::endl;
	std::cout << "Duration symsum_forward_unoptimized: " << (std::chrono::duration_cast<std::chrono::microseconds>(default_end - default_start).count() / 1000000.0) << " s" << std::endl;

	std::cout << "Max error: " << (output_avx - output_default).abs().max() << std::endl;
}