text:
namespace et {
namespace edit {

bool glTF::ParseGLTFData(const std::vector<uint8>& binaryContent, const std::string path, const std::string& extension, glTFAsset& asset)
{
	asset = glTFAsset();
	asset.basePath = path;

	std::string lowerExt;
	for (char val : extension)
	{
		if (val >= 'A' && val <= 'Z')
			lowerExt.push_back(val - ('A' - 'a'));
		else lowerExt.push_back(val);
	}
	if (lowerExt == "glb")
	{
		core::BinaryReader binReader;
		binReader.Open(binaryContent);
		if (!binReader.Exists())
		{
			LOG("glTF Failed to read the assetFile!", core::LogLevel::Warning);
			return false;
		}

		//Parse Header
		if (!ParseGLBHeader(&binReader, asset.header))
		{
			return false;
		}

		//Parse structured Json
		Chunk jsonChunk = Chunk();
		if (!ParseGLBChunk(&binReader, jsonChunk))
		{
			LOG("glTF failed to read json chunk from glb!", core::LogLevel::Warning);
			return false;
		}

		if (!(jsonChunk.chunkType == Chunk::ChunkType::JSON))
		{
			LOG("expected chunk type to be JSON", core::LogLevel::Warning);
			return false;
		}

		core::JSON::Parser parser = core::JSON::Parser(core::FileUtil::AsText(jsonChunk.chunkData));
		core::JSON::Object* root = parser.GetRoot();
		if (root == nullptr)return false;
		if (!glTF::ParseGlTFJson(root, asset.dom))return false;

		//Parse binary chunks
		while ((uint32)binReader.GetBufferPosition() < asset.header.length)
		{
			asset.dataChunks.push_back(Chunk());
			if (!ParseGLBChunk(&binReader, asset.dataChunks[asset.dataChunks.size() - 1]))
			{
				LOG("glTF failed to read binary chunk from glb!", core::LogLevel::Warning);
				return false;
			}

			if (!(asset.dataChunks[asset.dataChunks.size() - 1].chunkType == Chunk::ChunkType::BIN))
			{
				LOG("expected chunk type to be BIN", core::LogLevel::Warning);
				return false;
			}
		}

		return true;
	}
	else if (lowerExt == "gltf")
	{
		core::JSON::Parser parser = core::JSON::Parser(core::FileUtil::AsText(binaryContent));
		core::JSON::Object* root = parser.GetRoot();
		if (root == nullptr)return false;
		if (!glTF::ParseGlTFJson(root, asset.dom))return false;
		return true;
	}

	LOG("Unrecognized glTF extension", core::LogLevel::Warning);
	return false;
}

bool glTF::ParseGLBHeader(core::BinaryReader* binReader, Header &header)
{
	header.magic = binReader->Read<uint32>();
	if (!(header.magic == *reinterpret_cast<uint32*>("glTF")))
	{
		LOG("invalid glb file header!", core::LogLevel::Warning);
		return false;
	}
	header.version = binReader->Read<uint32>();
	if (!(header.version == 2))
	{
		LOG("invalid glb file header version!", core::LogLevel::Warning);
		return false;
	}
	header.length = binReader->Read<uint32>();
	return true;
}

bool glTF::ParseGLBChunk(core::BinaryReader* binReader, Chunk &chunk)
{
	if (binReader->GetBufferPosition() % 4u != 0u)//Make sure 4 byte alignment is respected
	{
		LOG("Expected binary buffer position for glb to be 4 byte aligned", core::LogLevel::Warning);
		binReader->SetBufferPosition(((binReader->GetBufferPosition() / 4u) + 1u) * 4u);
	}

	chunk.chunkLength = binReader->Read<uint32>();
	chunk.chunkType = static_cast<Chunk::ChunkType>(binReader->Read<uint32>());
	chunk.chunkData.reserve(chunk.chunkLength);
	for (uint32 i = 0; i < chunk.chunkLength; i++)
	{
		chunk.chunkData.push_back(binReader->Read<uint8>());
	}

	if (binReader->GetBufferPosition() % 4u != 0u)//Make sure 4 byte alignment is respected
	{
		binReader->SetBufferPosition(((binReader->GetBufferPosition() / 4u) + 1u) * 4u);
	}

	return true;
}

bool glTF::ParseGlTFJson(core::JSON::Object* json, Dom& dom)
{
	//Minimum requirement: asset and version
	if (!ParseAssetJson(json, dom.asset))
	{
		LOG("Failed to parse assset from JSON, invalid glTF file", core::LogLevel::Warning);
		return false;
	}
	//Also parse extensions and make sure we support them
	if (!ParseExtensionsJson(json, dom))
	{
		LOG("Failed to parse extensions from JSON, invalid glTF file", core::LogLevel::Warning);
		return false;
	}

	//Rest of the data we are interested in
	if (!ParseSceneJson(json, dom))
	{
		LOG("Failed to parse glTF scene from JSON", core::LogLevel::Warning);
		return false;
	}
	if (!ParseScenesJson(json, dom.scenes))
	{
		LOG("Failed to parse glTF scenes from JSON", core::LogLevel::Warning);
		return false;
	}
	if (!ParseNodesJson(json, dom.nodes))
	{
		LOG("Failed to parse glTF nodes from JSON", core::LogLevel::Warning);
		return false;
	}
	if (!ParseMeshesJson(json, dom.meshes))
	{
		LOG("Failed to parse glTF meshes from JSON", core::LogLevel::Warning);
		return false;
	}
	if (!ParseAccessorsJson(json, dom.accessors))
	{
		LOG("Failed to parse glTF accessors from JSON", core::LogLevel::Warning);
		return false;
	}
	if (!ParseBufferViewsJson(json, dom.bufferViews))
	{
		LOG("Failed to parse glTF bufferViews from JSON", core::LogLevel::Warning);
		return false;
	}
	if (!ParseBuffersJson(json, dom.buffers))
	{
		LOG("Failed to parse glTF buffers from JSON", core::LogLevel::Warning);
		return false;
	}
	if (!ParseImagesJson(json, dom.images))
	{
		LOG("Failed to parse glTF images from JSON", core::LogLevel::Warning);
		return false;
	}
	if (!ParseTexturesJson(json, dom.textures))
	{
		LOG("Failed to parse glTF textures from JSON", core::LogLevel::Warning);
		return false;
	}
	if (!ParseSamplersJson(json, dom.samplers))
	{
		LOG("Failed to parse glTF samplers from JSON", core::LogLevel::Warning);
		return false;
	}
	if (!ParseMaterialsJson(json, dom.materials))
	{
		LOG("Failed to parse glTF materials from JSON", core::LogLevel::Warning);
		return false;
	}
	if (!ParseCamerasJson(json, dom.cameras))
	{
		LOG("Failed to parse glTF cameras from JSON", core::LogLevel::Warning);
		return false;
	}
	if (!ParseSkinsJson(json, dom.skins))
	{
		LOG("Failed to parse glTF skins from JSON", core::LogLevel::Warning);
		return false;
	}
	if (!ParseAnimationsJson(json, dom.animations))
	{
		LOG("Failed to parse glTF animations from JSON", core::LogLevel::Warning);
		return false;
	}

	return true;
}

bool glTF::ParseAssetJson(core::JSON::Object* root, Asset& asset)
{
	core::JSON::Value* assetVal = (*root)["asset"];
	if (!assetVal)return false;
	if (!(assetVal->GetType() == core::JSON::ValueType::JSON_Object)) return false;
	core::JSON::Object* assetObj = assetVal->obj();

	if (!core::JSON::ApplyStrValue(assetObj, asset.version, "version")) return false;
	core::JSON::ApplyStrValue(assetObj, asset.minVersion, "minVersion");
	core::JSON::ApplyStrValue(assetObj, asset.generator, "generator");
	core::JSON::ApplyStrValue(assetObj, asset.copyright, "copyright");

	//Check version support
	bool hasMinVersion = false;
	if (asset.minVersion.size())
	{
		hasMinVersion = true;
		float minAssetVersion = std::stof(asset.minVersion);
		if (minAssetVersion < glTF::minVersion || minAssetVersion > glTF::maxVersion)
		{
			LOG("glTF minVersion is not supported by ETEngine", core::LogLevel::Warning);
			LogGLTFVersionSupport();
			return false;
		}
	}
	float version = std::stof(asset.version);
	if (version < glTF::minVersion)
	{
		LOG("glTF version is too low not supported by ETEngine", core::LogLevel::Warning);
		LogGLTFVersionSupport();
		return false;
	}
	if (!hasMinVersion)
	{
		if (version > glTF::maxVersion)
		{
			LOG("glTF version is too high not supported by ETEngine", core::LogLevel::Warning);
			LogGLTFVersionSupport();
			return false;
		}
	}

	return true;
}

bool glTF::ParseExtensionsJson(core::JSON::Object* root, Dom& dom)
{
	core::JSON::Value* extUsedVal = (*root)["extensionsRequired"];
	if (extUsedVal)
	{
		if (!(extUsedVal->GetType() == core::JSON::ValueType::JSON_Array)) return false;
		dom.extensionsUsed = extUsedVal->arr()->StrArr();
	}

	core::JSON::Value* extRequVal = (*root)["extensionsRequired"];
	if (extRequVal)
	{
		if (!(extRequVal->GetType() == core::JSON::ValueType::JSON_Array)) return false;
		dom.extensionsRequired = extRequVal->arr()->StrArr();
	}

	//Check extension support
	for (auto requ : dom.extensionsRequired)
	{
		if (std::find(dom.extensionsUsed.begin(), dom.extensionsUsed.end(), requ) == dom.extensionsUsed.end())
		{
			LOG(std::string("Required glTF extension '") + requ + "' not found in used extensions", core::LogLevel::Warning);
			return false;
		}
		if (std::find(supportedExtensions.begin(), supportedExtensions.end(), requ) == supportedExtensions.end())
		{
			LOG(std::string("Required glTF extension '") + requ + "' is not supported by ETEngine", core::LogLevel::Warning);
			return false;
		}
	}
	return true;
}

bool glTF::ParseSceneJson(core::JSON::Object* root, Dom& dom)
{
	core::JSON::ApplyIntValue(root, dom.scene, "scene");
	return true;
}

bool glTF::ParseScenesJson(core::JSON::Object* root, std::vector<Scene>& scenes)
{
	core::JSON::Value* scenesVal = (*root)["scenes"];
	if (!scenesVal)return true;

	if (!(scenesVal->GetType() == core::JSON::ValueType::JSON_Array)) return false;
	core::JSON::Array* scenesArr = scenesVal->arr();

	for (core::JSON::Value* sceneVal : scenesArr->value)
	{
		if (!(sceneVal->GetType() == core::JSON::ValueType::JSON_Object)) return false;
		core::JSON::Object* sceneObj = sceneVal->obj();

		Scene scene;

		core::JSON::ApplyStrValue(sceneObj, scene.name, "name");

		core::JSON::Value* nodesVal = (*sceneObj)["nodes"];
		if (nodesVal)
		{
			if (!(nodesVal->GetType() == core::JSON::ValueType::JSON_Array)) return false;
			core::JSON::Array* nodesArr = nodesVal->arr();
			std::vector<int64> intArr = nodesArr->IntArr();
			for (auto el : intArr) scene.nodes.push_back(static_cast<uint32>(el));
		}

		scenes.push_back(scene);
	}

	return true;
}

bool glTF::ParseNodesJson(core::JSON::Object* root, std::vector<Node>& nodes)
{
	core::JSON::Value* nodesVal = (*root)["nodes"];
	if (!nodesVal)return true;

	if (!(nodesVal->GetType() == core::JSON::ValueType::JSON_Array)) return false;
	core::JSON::Array* nodesArr = nodesVal->arr();

	for (core::JSON::Value* nodeVal : nodesArr->value)
	{
		if (!(nodeVal->GetType() == core::JSON::ValueType::JSON_Object)) return false;
		core::JSON::Object* nodeObj = nodeVal->obj();

		Node node;

		core::JSON::ApplyStrValue(nodeObj, node.name, "name");

		core::JSON::Value* childrenVal = (*nodeObj)["children"];
		if (childrenVal)
		{
			if (!(childrenVal->GetType() == core::JSON::ValueType::JSON_Array)) return false;
			core::JSON::Array* childrenArr = childrenVal->arr();
			std::vector<int64> intArr = childrenArr->IntArr();
			for (auto el : intArr) node.children.push_back(static_cast<uint32>(el));
		}

		bool hasTranslation = false;
		core::JSON::Value* translationVal = (*nodeObj)["translation"];
		if (translationVal)
		{
			if (!core::JSON::ArrayVector(translationVal, node.translation))return false;
			hasTranslation = true;
		}

		bool hasRotation = false;
		core::JSON::Value* rotationVal = (*nodeObj)["rotation"];
		if (rotationVal)
		{
			if (!core::JSON::ArrayVector(rotationVal, node.rotation.v4))return false;
			hasRotation = true;
		}

		bool hasScale = false;
		core::JSON::Value* scaleVal = (*nodeObj)["scale"];
		if (scaleVal)
		{
			if (!core::JSON::ArrayVector(scaleVal, node.scale))return false;
			hasScale = true;
		}

		bool hasMatrix = false;
		core::JSON::Value* matrixVal = (*nodeObj)["matrix"];
		if (matrixVal)
		{
			if (!core::JSON::ArrayMatrix(matrixVal, node.matrix))return false;
			hasMatrix = true;
		}

		if (hasMatrix)
		{
			vec3 trans, scale;
			quat rot;
			math::decomposeTRS(node.matrix, trans, rot, scale);
			if (!hasTranslation)node.translation = trans;
			else if (!math::nearEqualsV(node.translation, trans, 0.0001f))LOG("inconsistent translation values for node", core::LogLevel::Warning);
			if (!hasRotation)node.rotation = rot;
			else
			{
				//different quaternions can express the same rotation
				vec3 testVec = vec3(0, 0, 1);
				if (!math::nearEqualsV(node.rotation * testVec, rot * testVec, 0.0001f))LOG("inconsistent rotation values for node", core::LogLevel::Warning);
			}
			if (!hasScale)node.scale = scale;
			else if (!math::nearEqualsV(node.scale, scale, 0.0001f))LOG("inconsistent scale values for node", core::LogLevel::Warning);
		}
		else
		{
			node.matrix = math::scale(node.scale) * math::rotate(node.rotation) * math::translate(node.translation);
		}

		core::JSON::ApplyIntValue(nodeObj, node.camera, "camera");
		core::JSON::ApplyIntValue(nodeObj, node.mesh, "mesh");
		core::JSON::ApplyIntValue(nodeObj, node.skin, "skin");

		core::JSON::Value* weightsVal = (*nodeObj)["weights"];
		if (weightsVal)
		{
			if (!(weightsVal->GetType() == core::JSON::ValueType::JSON_Array)) return false;
			core::JSON::Array* weightsArr = weightsVal->arr();
			std::vector<double> arr = weightsArr->NumArr();
			for (auto el : arr) node.weights.push_back(static_cast<float>(el));
		}

		nodes.push_back(node);
	}

	return true;
}

bool glTF::ParseMeshesJson(core::JSON::Object* root, std::vector<Mesh>& meshes)
{
	core::JSON::Value* meshesVal = (*root)["meshes"];
	if (!meshesVal)return true;

	if (!(meshesVal->GetType() == core::JSON::ValueType::JSON_Array)) return false;
	core::JSON::Array* meshesArr = meshesVal->arr();

	for (core::JSON::Value* meshVal : meshesArr->value)
	{
		if (!(meshVal->GetType() == core::JSON::ValueType::JSON_Object)) return false;
		core::JSON::Object* meshObj = meshVal->obj();

		Mesh mesh;

		core::JSON::ApplyStrValue(meshObj, mesh.name, "name");

		core::JSON::Value* primitivesVal = (*meshObj)["primitives"];
		if (primitivesVal)
		{
			if (!(primitivesVal->GetType() == core::JSON::ValueType::JSON_Array)) return false;
			core::JSON::Array* primitivesArr = primitivesVal->arr();
			for (core::JSON::Value* primitiveVal : primitivesArr->value)
			{
				if (!(primitiveVal->GetType() == core::JSON::ValueType::JSON_Object)) return false;
				core::JSON::Object* primitiveObj = primitiveVal->obj();
				Primitive prim;
				if (ParsePrimitiveJson(primitiveObj, prim))
				{
					mesh.primitives.push_back(prim);
				}
				else return false;
			}
		}

		core::JSON::Value* weightsVal = (*meshObj)["weights"];
		if (weightsVal)
		{
			if (!(weightsVal->GetType() == core::JSON::ValueType::JSON_Array)) return false;
			core::JSON::Array* nodesArr = weightsVal->arr();
			std::vector<double> weightArr = nodesArr->NumArr();
			for (auto el : weightArr) mesh.weights.push_back(static_cast<float>(el));
		}

		meshes.push_back(mesh);
	}

	return true;
}

bool glTF::ParsePrimitiveJson(core::JSON::Object* primitiveObj, Primitive& primitive)
{
	core::JSON::Value* attributesVal = (*primitiveObj)["attributes"];
	if (!attributesVal)return false;
	if (!(attributesVal->GetType() == core::JSON::ValueType::JSON_Object)) return false;
	core::JSON::Object* attributeObj = attributesVal->obj();
	core::JSON::ApplyIntValue(attributeObj, primitive.attributes.position, "POSITION");
	core::JSON::ApplyIntValue(attributeObj, primitive.attributes.normal, "NORMAL");
	core::JSON::ApplyIntValue(attributeObj, primitive.attributes.tangent, "TANGENT");
	core::JSON::ApplyIntValue(attributeObj, primitive.attributes.texcoord0, "TEXCOORD_0");
	core::JSON::ApplyIntValue(attributeObj, primitive.attributes.texcoord1, "TEXCOORD_1");
	core::JSON::ApplyIntValue(attributeObj, primitive.attributes.color0, "COLOR_0");
	core::JSON::ApplyIntValue(attributeObj, primitive.attributes.joints0, "JOINTS_0");
	core::JSON::ApplyIntValue(attributeObj, primitive.attributes.weights0, "WEIGHTS_0");

	core::JSON::ApplyIntValue(primitiveObj, primitive.indices, "indices");
	core::JSON::ApplyIntValue(primitiveObj, primitive.material, "material");
	core::JSON::ApplyIntValue(primitiveObj, primitive.mode, "mode");

	core::JSON::Value* targetsVal = (*primitiveObj)["targets"];
	if (targetsVal)
	{
		if (!(targetsVal->GetType() == core::JSON::ValueType::JSON_Array)) return false;
		core::JSON::Array* targetsArr = targetsVal->arr();
		for (core::JSON::Value* targetVal : targetsArr->value)
		{
			if (!(targetVal->GetType() == core::JSON::ValueType::JSON_Object)) return false;
			core::JSON::Object* targetObj = targetVal->obj();

			Primitive::Targets target;

			core::JSON::ApplyIntValue(targetObj, target.position, "POSITION");
			core::JSON::ApplyIntValue(targetObj, target.normal, "NORMAL");
			core::JSON::ApplyIntValue(targetObj, target.tangent, "TANGENT");

			primitive.targets.push_back(target);
		}
	}

	return true;
}

bool glTF::ParseAccessorsJson(core::JSON::Object* root, std::vector<Accessor>& accessors)
{
	using namespace core;

	JSON::Value* accessorsVal = (*root)["accessors"];
	if (!accessorsVal)return true;

	if (!(accessorsVal->GetType() == JSON::ValueType::JSON_Array)) return false;
	JSON::Array* accessorsArr = accessorsVal->arr();

	for (JSON::Value* accessorVal : accessorsArr->value)
	{
		if (!(accessorVal->GetType() == JSON::ValueType::JSON_Object)) return false;
		JSON::Object* accessorObj = accessorVal->obj();

		Accessor accessor;

		//Required components
		if (!JSON::ApplyIntValue(accessorObj, accessor.componentType, "componentType"))return false;
		if (!JSON::ApplyIntValue(accessorObj, accessor.count, "count"))return false;
		std::string typeStr;
		if (!JSON::ApplyStrValue(accessorObj, typeStr, "type"))return false;
		bool typeFound = false;
		for (auto const& type : AccessorTypes)
		{
			if (type.second.second == typeStr)
			{
				accessor.type = type.first;
				typeFound = true;
				break;
			}
		}
		if (!typeFound)return false;

		//Rest
		JSON::ApplyIntValue(accessorObj, accessor.bufferView, "bufferView");
		JSON::ApplyIntValue(accessorObj, accessor.byteOffset, "byteOffset");
		JSON::ApplyBoolValue(accessorObj, accessor.normalized, "normalized");
		JSON::ApplyStrValue(accessorObj, accessor.name, "name");

		JSON::Value* maxVal = (*accessorObj)["max"];
		if (maxVal)
		{
			if (!(maxVal->GetType() == JSON::ValueType::JSON_Array)) return false;
			JSON::Array* jArr = maxVal->arr();
			std::vector<double> arr = jArr->NumArr();
			for (auto el : arr) accessor.max.push_back(static_cast<float>(el));
		}
		JSON::Value* minVal = (*accessorObj)["min"];
		if (minVal)
		{
			if (!(minVal->GetType() == JSON::ValueType::JSON_Array)) return false;
			JSON::Array* jArr = minVal->arr();
			std::vector<double> arr = jArr->NumArr();
			for (auto el : arr) accessor.min.push_back(static_cast<float>(el));
		}

		accessors.push_back(accessor);

		JSON::Value* sparseVal = (*accessorObj)["sparse"];
		if (sparseVal)
		{
			if (!(sparseVal->GetType() == JSON::ValueType::JSON_Object)) return false;
			JSON::Object* sparseObj = sparseVal->obj();

			accessors[accessors.size() - 1].sparse = new Accessor::Sparse();
			Accessor::Sparse* sparse = accessors[accessors.size() - 1].sparse;

			if (!JSON::ApplyIntValue(sparseObj, sparse->count, "count"))return false;

			JSON::Value* indicesVal = (*sparseObj)["indices"];
			if (!indicesVal)return false;
			if (!(indicesVal->GetType() == JSON::ValueType::JSON_Object)) return false;
			JSON::Object* indicesObj = indicesVal->obj();
			if (!JSON::ApplyIntValue(indicesObj, sparse->indices.bufferView, "bufferView"))return false;
			JSON::ApplyIntValue(indicesObj, sparse->indices.byteOffset, "byteOffset");
			if (!JSON::ApplyIntValue(indicesObj, sparse->indices.componentType, "componentType"))return false;

			JSON::Value* valuesVal = (*sparseObj)["values"];
			if (!valuesVal)return false;
			if (!(valuesVal->GetType() == JSON::ValueType::JSON_Object)) return false;
			JSON::Object* valuesObj = valuesVal->obj();
			if (!JSON::ApplyIntValue(valuesObj, sparse->values.bufferView, "bufferView"))return false;
			JSON::ApplyIntValue(valuesObj, sparse->values.byteOffset, "byteOffset");
		}
	}
	return true;
}

bool glTF::ParseBufferViewsJson(core::JSON::Object* root, std::vector<BufferView>& bufferViews)
{
	using namespace core;

	JSON::Value* bufferViewsVal = (*root)["bufferViews"];
	if (!bufferViewsVal)return true;

	if (!(bufferViewsVal->GetType() == JSON::ValueType::JSON_Array)) return false;
	JSON::Array* bufferViewsArr = bufferViewsVal->arr();

	for (JSON::Value* bufferViewVal : bufferViewsArr->value)
	{
		if (!(bufferViewVal->GetType() == JSON::ValueType::JSON_Object)) return false;
		JSON::Object* bufferViewObj = bufferViewVal->obj();

		BufferView view;

		if (!JSON::ApplyIntValue(bufferViewObj, view.buffer, "buffer"))return false;
		if (!JSON::ApplyIntValue(bufferViewObj, view.byteLength, "byteLength"))return false;
		JSON::ApplyIntValue(bufferViewObj, view.byteOffset, "byteOffset");
		JSON::ApplyIntValue(bufferViewObj, view.byteStride, "byteStride");
		JSON::ApplyIntValue(bufferViewObj, view.target, "target");
		JSON::ApplyStrValue(bufferViewObj, view.name, "name");

		bufferViews.push_back(view);
	}

	return true;
}

bool glTF::ParseBuffersJson(core::JSON::Object* root, std::vector<Buffer>& buffers)
{
	using namespace core;

	JSON::Value* buffersVal = (*root)["buffers"];
	if (!buffersVal)return true;

	if (!(buffersVal->GetType() == JSON::ValueType::JSON_Array)) return false;
	JSON::Array* buffersArr = buffersVal->arr();

	for (JSON::Value* bufferVal : buffersArr->value)
	{
		if (!(bufferVal->GetType() == JSON::ValueType::JSON_Object)) return false;
		JSON::Object* bufferObj = bufferVal->obj();

		Buffer buffer;

		JSON::Value const* const jval = (*bufferObj)["uri"];
		if (jval)
		{
			JSON::String const* const jstr = jval->str();
			if (jstr)
			{
				buffer.uri.SetPath(jstr->value);
			}
		}

		if (!JSON::ApplyIntValue(bufferObj, buffer.byteLength, "byteLength"))
		{
			return false;
		}

		JSON::ApplyStrValue(bufferObj, buffer.name, "name");

		buffers.push_back(buffer);
	}

	return true;
}

bool glTF::ParseTexturesJson(core::JSON::Object* root, std::vector<Texture>& textures)
{
	using namespace core;

	JSON::Value* texturesVal = (*root)["textures"];
	if (!texturesVal)return true;

	if (!(texturesVal->GetType() == JSON::ValueType::JSON_Array)) return false;
	JSON::Array* texturesArr = texturesVal->arr();

	for (JSON::Value* textureVal : texturesArr->value)
	{
		if (!(textureVal->GetType() == JSON::ValueType::JSON_Object)) return false;
		JSON::Object* textureObj = textureVal->obj();

		Texture texture;

		JSON::ApplyIntValue(textureObj, texture.sampler, "sampler");
		JSON::ApplyIntValue(textureObj, texture.source, "source");
		JSON::ApplyStrValue(textureObj, texture.name, "name");

		textures.push_back(texture);
	}

	return true;
}

bool glTF::ParseImagesJson(core::JSON::Object* root, std::vector<Image>& images)
{
	using namespace core;

	JSON::Value* imagesVal = (*root)["images"];
	if (!imagesVal)return true;

	if (!(imagesVal->GetType() == JSON::ValueType::JSON_Array)) return false;
	JSON::Array* imagesArr = imagesVal->arr();

	for (JSON::Value* imageVal : imagesArr->value)
	{
		if (!(imageVal->GetType() == JSON::ValueType::JSON_Object)) return false;
		JSON::Object* imageObj = imageVal->obj();

		Image image;

		JSON::Value const* const jval = (*imageObj)["uri"];
		if (jval)
		{
			JSON::String const* const jstr = jval->str();
			if (jstr)
			{
				image.uri.SetPath(jstr->value);
			}
		}

		JSON::ApplyIntValue(imageObj, image.bufferView, "bufferView");
		JSON::ApplyStrValue(imageObj, image.mimeType, "mimeType");
		JSON::ApplyStrValue(imageObj, image.name, "name");

		images.push_back(image);
	}

	return true;
}

bool glTF::ParseSamplersJson(core::JSON::Object* root, std::vector<Sampler>& samplers)
{
	using namespace core;

	JSON::Value* samplersVal = (*root)["samplers"];
	if (!samplersVal)return true;

	if (!(samplersVal->GetType() == JSON::ValueType::JSON_Array)) return false;
	JSON::Array* samplersArr = samplersVal->arr();

	for (JSON::Value* samplerVal : samplersArr->value)
	{
		if (!(samplerVal->GetType() == JSON::ValueType::JSON_Object)) return false;
		JSON::Object* samplerObj = samplerVal->obj();

		Sampler sampler;

		JSON::ApplyIntValue(samplerObj, sampler.magFilter, "magFilter");
		JSON::ApplyIntValue(samplerObj, sampler.minFilter, "minFilter");
		JSON::ApplyIntValue(samplerObj, sampler.wrapS, "wrapS");
		JSON::ApplyIntValue(samplerObj, sampler.wrapT, "wrapT");
		JSON::ApplyStrValue(samplerObj, sampler.name, "name");

		samplers.push_back(sampler);
	}

	return true;
}

bool glTF::ParseMaterialsJson(core::JSON::Object* root, std::vector<Material>& materials)
{
	using namespace core;

	JSON::Value* materialsVal = (*root)["materials"];
	if (!materialsVal)return true;

	if (!(materialsVal->GetType() == JSON::ValueType::JSON_Array)) return false;
	JSON::Array* materialsArr = materialsVal->arr();

	for (JSON::Value* materialVal : materialsArr->value)
	{
		if (!(materialVal->GetType() == JSON::ValueType::JSON_Object)) return false;
		JSON::Object* materialObj = materialVal->obj();

		materials.push_back(Material());
		Material* material = &materials[materials.size() - 1];

		JSON::ApplyStrValue(materialObj, material->name, "name");

		if (material->pbrMetallicRoughness)return false;
		material->pbrMetallicRoughness = new Material::PbrMetallicRoughness();
		JSON::Value* pbrMetallicRoughnessVal = (*materialObj)["pbrMetallicRoughness"];
		if (pbrMetallicRoughnessVal)
		{
			if (!(pbrMetallicRoughnessVal->GetType() == JSON::ValueType::JSON_Object)) return false;
			if (!ParsePbrMetallicRoughnessJson(pbrMetallicRoughnessVal->obj(), material->pbrMetallicRoughness))return false;
		}

		JSON::Value* normalTextureVal = (*materialObj)["normalTexture"];
		if (normalTextureVal)
		{
			if (!(normalTextureVal->GetType() == JSON::ValueType::JSON_Object)) return false;
			JSON::Object* normalTextureObj = normalTextureVal->obj();
			if (material->normalTexture)return false;
			material->normalTexture = new Material::NormalTextureInfo();
			if (!ParseTextureInfoJson(normalTextureObj, material->normalTexture))return false;
			if (material->normalTexture)
			{
				JSON::ApplyNumValue(normalTextureObj, material->normalTexture->scale, "scale");
			}
		}

		JSON::Value* occlusionTextureVal = (*materialObj)["occlusionTexture"];
		if (occlusionTextureVal)
		{
			if (!(occlusionTextureVal->GetType() == JSON::ValueType::JSON_Object)) return false;
			JSON::Object* occlusionTextureObj = occlusionTextureVal->obj();
			if (material->occlusionTexture)return false;
			material->occlusionTexture = new Material::OcclusionTextureInfo();
			if (!ParseTextureInfoJson(occlusionTextureObj, material->occlusionTexture))return false;
			if (material->occlusionTexture)
			{
				JSON::ApplyNumValue(occlusionTextureObj, material->occlusionTexture->strength, "strength");
			}
		}

		JSON::Value* emissiveTextureVal = (*materialObj)["emissiveTexture"];
		if (emissiveTextureVal)
		{
			if (!(emissiveTextureVal->GetType() == JSON::ValueType::JSON_Object)) return false;
			if (material->emissiveTexture)return false;
			material->emissiveTexture = new Material::TextureInfo();
			if (!ParseTextureInfoJson(emissiveTextureVal->obj(), material->emissiveTexture))return false;
		}

		JSON::Value* emissiveFactorVal = (*materialObj)["emissiveFactor"];
		if (emissiveFactorVal)
		{
			if (!JSON::ArrayVector(emissiveFactorVal, material->emissiveFactor))return false;
		}

		std::string alphaModeStr;
		if (JSON::ApplyStrValue(materialObj, alphaModeStr, "alphaMode"))
		{
			bool alphaModeFound = false;
			for (auto const& alphaMode : AlphaModes)
			{
				if (alphaMode.second == alphaModeStr)
				{
					material->alphaMode = alphaMode.first;
					alphaModeFound = true;
					break;
				}
			}
			if (!alphaModeFound)return false;
		}

		JSON::ApplyNumValue(materialObj, material->alphaCutoff, "alphaCutoff");
		JSON::ApplyBoolValue(materialObj, material->doubleSided, "doubleSided");
	}

	return true;
}

bool glTF::ParsePbrMetallicRoughnessJson(core::JSON::Object* pbrObj, Material::PbrMetallicRoughness* pbr)
{
	using namespace core;

	if (!pbr) return false;

	JSON::Value* baseColorFactorVal = (*pbrObj)["baseColorFactor"];
	if (baseColorFactorVal)
	{
		if (!JSON::ArrayVector(baseColorFactorVal, pbr->baseColorFactor))return false;
	}

	JSON::Value* baseColorTextureVal = (*pbrObj)["baseColorTexture"];
	if (baseColorTextureVal)
	{
		if (!(baseColorTextureVal->GetType() == JSON::ValueType::JSON_Object)) return false;
		if (pbr->baseColorTexture)return false;
		pbr->baseColorTexture = new Material::TextureInfo();
		if (!ParseTextureInfoJson(baseColorTextureVal->obj(), pbr->baseColorTexture))return false;
	}

	JSON::ApplyNumValue(pbrObj, pbr->metallicFactor, "metallicFactor");

	JSON::ApplyNumValue(pbrObj, pbr->roughnessFactor, "roughnessFactor");

	JSON::Value* metallicRoughnessTextureVal = (*pbrObj)["metallicRoughnessTexture"];
	if (metallicRoughnessTextureVal)
	{
		if (!(metallicRoughnessTextureVal->GetType() == JSON::ValueType::JSON_Object)) return false;
		if (pbr->metallicRoughnessTexture)return false;
		pbr->metallicRoughnessTexture = new Material::TextureInfo();
		if (!ParseTextureInfoJson(metallicRoughnessTextureVal->obj(), pbr->metallicRoughnessTexture))return false;
	}

	return true;
}

bool glTF::ParseTextureInfoJson(core::JSON::Object* textureInfo, Material::TextureInfo* info)
{
	using namespace core;

	if (!info)return false;

	if (!JSON::ApplyIntValue(textureInfo, info->index, "index"))return false;
	JSON::ApplyIntValue(textureInfo, info->texCoord, "texCoord");

	return true;
}

bool glTF::ParseCamerasJson(core::JSON::Object* root, std::vector<Camera>& cameras)
{
	using namespace core;

	JSON::Value* camerasVal = (*root)["cameras"];
	if (!camerasVal)return true;

	if (!(camerasVal->GetType() == JSON::ValueType::JSON_Array)) return false;
	JSON::Array* camerasArr = camerasVal->arr();

	for (JSON::Value* cameraVal : camerasArr->value)
	{
		if (!(cameraVal->GetType() == JSON::ValueType::JSON_Object)) return false;
		JSON::Object* cameraObj = cameraVal->obj();

		cameras.push_back(Camera());
		Camera* camera = &cameras[cameras.size() - 1];

		std::string cameraTypeStr;
		if (!(JSON::ApplyStrValue(cameraObj, cameraTypeStr, "type")))return false;
		bool cameraTypeFound = false;
		for (auto const& cameraType : CameraTypes)
		{
			if (cameraType.second == cameraTypeStr)
			{
				camera->type = cameraType.first;
				cameraTypeFound = true;
				break;
			}
		}
		if (!cameraTypeFound)return false;
		JSON::ApplyStrValue(cameraObj, camera->name, "name");

		JSON::Value* orthographicVal = (*cameraObj)["orthographic"];
		if (orthographicVal)
		{
			if (!(orthographicVal->GetType() == JSON::ValueType::JSON_Object)) return false;
			JSON::Object* orthographicObj = orthographicVal->obj();

			if (camera->orthographic)return false;
			camera->orthographic = new Camera::Orthographic();

			if (!JSON::ApplyNumValue(orthographicObj, camera->orthographic->xmag, "xmag"))return false;
			if (!JSON::ApplyNumValue(orthographicObj, camera->orthographic->yamg, "yamg"))return false;
			if (!JSON::ApplyNumValue(orthographicObj, camera->orthographic->zfar, "zfar"))return false;
			if (!JSON::ApplyNumValue(orthographicObj, camera->orthographic->znear, "znear"))return false;
		}

		JSON::Value* perspectiveVal = (*cameraObj)["perspective"];
		if (perspectiveVal)
		{
			if (!(perspectiveVal->GetType() == JSON::ValueType::JSON_Object)) return false;
			JSON::Object* perspectiveObj = perspectiveVal->obj();

			if (camera->perspective)return false;
			camera->perspective = new Camera::Perspective();

			JSON::ApplyNumValue(perspectiveObj, camera->perspective->aspectRatio, "aspectRatio");
			if (!JSON::ApplyNumValue(perspectiveObj, camera->perspective->yfov, "yfov"))return false;
			JSON::ApplyNumValue(perspectiveObj, camera->perspective->zfar, "zfar");
			if (!JSON::ApplyNumValue(perspectiveObj, camera->perspective->znear, "znear"))return false;
		}
	}

	return true;
}

bool glTF::ParseSkinsJson(core::JSON::Object* root, std::vector<Skin>& skins)
{
	using namespace core;

	JSON::Value* skinsVal = (*root)["skins"];
	if (!skinsVal)return true;

	if (!(skinsVal->GetType() == JSON::ValueType::JSON_Array)) return false;
	JSON::Array* skinsArr = skinsVal->arr();

	for (JSON::Value* skinVal : skinsArr->value)
	{
		if (!(skinVal->GetType() == JSON::ValueType::JSON_Object)) return false;
		JSON::Object* skinObj = skinVal->obj();

		Skin skin;

		JSON::Value* jointsVal = (*skinObj)["skins"];
		if (!jointsVal)return false;
		if (!(jointsVal->GetType() == JSON::ValueType::JSON_Array)) return false;
		JSON::Array* jointsArr = jointsVal->arr();
		std::vector<int64> intArr = jointsArr->IntArr();
		for (auto el : intArr) skin.joints.push_back(static_cast<uint32>(el));

		JSON::ApplyStrValue(skinObj, skin.name, "name");

		JSON::ApplyIntValue(skinObj, skin.inverseBindMatrices, "inverseBindMatrices");
		JSON::ApplyIntValue(skinObj, skin.skeleton, "skeleton");

		skins.push_back(skin);
	}

	return true;
}

bool glTF::ParseAnimationsJson(core::JSON::Object* root, std::vector<Animation>& animations)
{
	using namespace core;

	JSON::Value* animationsVal = (*root)["animations"];
	if (!animationsVal)return true;

	if (!(animationsVal->GetType() == JSON::ValueType::JSON_Array)) return false;
	JSON::Array* animationsArr = animationsVal->arr();

	for (JSON::Value* animationVal : animationsArr->value)
	{
		if (!(animationVal->GetType() == JSON::ValueType::JSON_Object)) return false;
		JSON::Object* animationObj = animationVal->obj();

		Animation animation;

		JSON::ApplyStrValue(animationObj, animation.name, "name");

		JSON::Value* channelsVal = (*animationObj)["channels"];
		if (!channelsVal)return false;
		if (!(channelsVal->GetType() == JSON::ValueType::JSON_Array)) return false;
		JSON::Array* channelsArr = channelsVal->arr();
		for (JSON::Value* channelVal : channelsArr->value)
		{
			if (!(channelVal->GetType() == JSON::ValueType::JSON_Object)) return false;
			JSON::Object* channelObj = channelVal->obj();

			Animation::Channel channel;

			if (!JSON::ApplyIntValue(channelObj, channel.sampler, "sampler"))return false;

			JSON::Value* targetVal = (*channelObj)["target"];
			if (!targetVal)return false;
			if (!(targetVal->GetType() == JSON::ValueType::JSON_Object)) return false;
			JSON::Object* targetObj = targetVal->obj();

			JSON::ApplyIntValue(targetObj, channel.target.node, "node");

			std::string targetPathStr;
			if (!(JSON::ApplyStrValue(targetObj, targetPathStr, "path")))return false;
			bool targetPathFound = false;
			for (auto const& targetPath : ChannelTargetPaths)
			{
				if (targetPath.second == targetPathStr)
				{
					channel.target.path = targetPath.first;
					targetPathFound = true;
					break;
				}
			}
			if (!targetPathFound)return false;

			animation.channels.push_back(channel);
		}

		JSON::Value* samplersVal = (*animationObj)["samplers"];
		if (!samplersVal)return false;
		if (!(samplersVal->GetType() == JSON::ValueType::JSON_Array)) return false;
		JSON::Array* samplersArr = samplersVal->arr();
		for (JSON::Value* samplerVal : samplersArr->value)
		{
			if (!(samplerVal->GetType() == JSON::ValueType::JSON_Object)) return false;
			JSON::Object* samplerObj = samplerVal->obj();

			Animation::Sampler sampler;

			if (!JSON::ApplyIntValue(samplerObj, sampler.input, "input"))return false;
			if (!JSON::ApplyIntValue(samplerObj, sampler.output, "output"))return false;

			std::string samplerInterpolationStr;
			if (JSON::ApplyStrValue(samplerObj, samplerInterpolationStr, "interpolation"))
			{
				bool samplerInterpolationFound = false;
				for (auto const& samplerInterpolation : SamplerInterpolations)
				{
					if (samplerInterpolation.second == samplerInterpolationStr)
					{
						sampler.interpolation = samplerInterpolation.first;
						samplerInterpolationFound = true;
						break;
					}
				}
				if (!samplerInterpolationFound)return false;
			}

			animation.samplers.push_back(sampler);
		}

		animations.push_back(animation);
	}

	return true;
}

void glTF::LogGLTFVersionSupport()
{
	LOG(std::string("glTF minVersion ") + std::to_string(glTF::minVersion) + " maxVersion " + std::to_string(glTF::maxVersion));
}

bool glTF::OpenBufferViewReader(glTFAsset& asset, uint32 viewIdx, core::BinaryReader* pViewReader)
{
	if (viewIdx >= (int32)asset.dom.bufferViews.size())
	{
		LOG("BufferView index out of range", core::LogLevel::Warning);
		return false;
	}

	BufferView& view = asset.dom.bufferViews[viewIdx];
	if (view.buffer >= (uint32)asset.dom.buffers.size())
	{
		LOG("Buffer index out of range", core::LogLevel::Warning);
		return false;
	}

	Buffer& buffer = asset.dom.buffers[view.buffer];
	if (!buffer.uri.IsEvaluated())
	{
		if (!buffer.uri.Evaluate(asset.basePath))
		{
			LOG("Failed to evaluate buffer URI", core::LogLevel::Warning);
			return false;
		}
	}

	if (buffer.uri.GetType() == core::URI::E_Type::None)
	{
		if (view.buffer >= static_cast<uint32>(asset.dataChunks.size()))
		{
			LOG("No data chunk loaded for glb buffer", core::LogLevel::Warning);
			return false;
		}

		pViewReader->Open(asset.dataChunks[view.buffer].chunkData, static_cast<size_t>(view.byteOffset), static_cast<size_t>(view.byteLength));
	}
	else
	{
		pViewReader->Open(buffer.uri.GetEvaluatedData(), static_cast<size_t>(view.byteOffset), static_cast<size_t>(view.byteLength));
	}

	if (!pViewReader->Exists())
	{
		LOG("glTF Failed to read the buffer view!", core::LogLevel::Warning);
		return false;
	}

	return true;
}

bool glTF::GetAccessorData(glTFAsset& asset, uint32 idx, std::vector<uint8>& data)
{
	if (idx >= (uint32)asset.dom.accessors.size())
	{
		LOG("Accessor index out of range", core::LogLevel::Warning);
		return false;
	}
	Accessor& accessor = asset.dom.accessors[idx];
	if (accessor.sparse || accessor.bufferView == -1)
	{
		LOG("Unsupported accessor type, sparse accessors are not yet implemented", core::LogLevel::Warning);
		return false;
	}
	if (accessor.bufferView >= (int32)asset.dom.bufferViews.size())
	{
		LOG("BufferView index out of range", core::LogLevel::Warning);
		return false;
	}
	BufferView& view = asset.dom.bufferViews[accessor.bufferView];
	if (view.buffer >= (uint32)asset.dom.buffers.size())
	{
		LOG("Buffer index out of range", core::LogLevel::Warning);
		return false;
	}

	uint8 compSize = ComponentTypes[accessor.componentType];
	uint8 compsPerEl = AccessorTypes[accessor.type].first;
	uint8 elSize = compSize * compsPerEl;

	//Validation
	if (!(accessor.byteOffset % compSize == 0)) LOG("Accessors byte offset needs to be a multiple of the component size", core::LogLevel::Warning);
	if (accessor.min.size())
	{
		if (!((uint32)accessor.min.size() == (uint32)compsPerEl)) LOG("Accessors min array size must equal components per element", core::LogLevel::Warning);
	}
	if (accessor.max.size())
	{
		if (!((uint32)accessor.max.size() == (uint32)compsPerEl)) LOG("Accessors max array size must equal components per element", core::LogLevel::Warning);
	}

	uint32 stride = (view.byteStride == -1) ? (uint32)elSize : view.byteStride;
	if (!(stride % compSize == 0))
	{
		LOG("Accessors byte stride needs to be a multiple of the component size", core::LogLevel::Warning);
	}

	if ((accessor.byteOffset + stride * (accessor.count - 1) + elSize) > view.byteLength)
	{
		LOG("Accessors doesn't fit buffer view", core::LogLevel::Warning);
	}

	core::BinaryReader* pViewReader = new core::BinaryReader();
	if (!OpenBufferViewReader(asset, accessor.bufferView, pViewReader))
	{
		delete pViewReader;
		LOG("Unable to read buffer view", core::LogLevel::Warning);
		return false;
	}

	for (uint64 i = static_cast<uint64>(accessor.byteOffset);
		i < static_cast<uint64>(accessor.byteOffset) + (stride * accessor.count);
		i += static_cast<uint64>(stride))
	{
		pViewReader->SetBufferPosition(static_cast<size_t>(i));
		for (uint32 j = 0; j < elSize; ++j)
		{
			data.push_back(pViewReader->Read<uint8>());
		}
	}

	delete pViewReader;
	return true;
}

} // namespace edit
} // namespace et


text:
    Kavraki Lab  •
    Department of Computer Science •
    Rice University

    Funded in part by the National Science Foundation

    Documentation generated by doxygen 1.8.18

redpajama_set_name: RedPajamaGithub


text:
"""
This program reads BAM files and counts the number of reads that match
the alternate and reference allele at every SNP position in the provided
SNP HDF5 data files. The read counts are stored in specified HDF5 output
files.

Additionally counts of all reads are stored in another track (at the
left-most position of the reads).

This program does not perform filtering of reads based on mappability.
It is assumed that the inpute BAM files are filtered appropriately prior to
calling this script.

Reads that overlap known indels are not included in allele-specific
counts.

usage: bam2h5.py OPTIONS BAM_FILE1 [BAM_FILE2 ...]

BAM Files:
     Aligned reads are read from one or more BAM files. The provided
     BAM files must be sorted and indexed.

Input Options:
     --chrom CHROM_TXT_FILE [required]
	   Path to chromInfo.txt file (may be gzipped) with list of
	   chromosomes for the relevant genome assembly. Each line
	   in file should contain tab-separated chromosome name and
	   chromosome length (in basepairs). chromInfo.txt files can
	   be downloaded from the UCSC genome browser. For example,
	   a chromInfo.txt.gz file for hg19 can be downloaded from
	   http://hgdownload.soe.ucsc.edu/goldenPath/hg19/database/

     --snp_index SNP_INDEX_H5_FILE [required]
       Path to HDF5 file containing SNP index. The SNP index is
       used to convert the genomic position of a SNP to its
       corresponding row in the haplotype and snp_tab
       HDF5 files.

     --snp_tab SNP_TABLE_H5_FILE [required]
       Path to HDF5 file to read SNP information from. Each row of SNP
       table contains SNP name (rs_id), position, allele1, allele2.

     --haplotype HAPLOTYPE_H5_FILE [optional]
       Path to HDF5 file to read phased haplotypes from.
       If supplied, when read overlaps multiple SNPs counts are randomly
       assigned to ONE of the overlapping HETEROZYGOUS SNPs; if not supplied
       counts are randomly assigned to ONE of overlapping SNPs (regardless of
       their genotype).

     --samples SAMPLES_TXT_FILE [optional]
       Path to text file containing a list of individual identifiers. The
       ordering of individuals must be consistent with the haplotype
       file. The samples file is assumed to have one identifier per line
       in the first column (other columns are ignored).

     --individual INDIVIDUAL [optional]
       Identifier for individual, used to determine which
       SNPs are heterozygous. Must be provided
       if --haplotype argument is provided and must match one of the
       individuals in the file provided with --samples argument.

Output Options:
     --data_type uint8|uint16
       Data type of stored counts; uint8 takes up less disk
       space but has a maximum value of 255 (default=uint8).

     --ref_as_counts REF_AS_COUNT_H5_FILE [required]
       Path to HDF5 file to write counts of reads that match reference allele.
       Allele-specific counts are stored at the position of the SNP.

     --alt_as_counts ALT_AS_COUNT_H5_FILE [required]
       Path to HDF5 file to write counts of reads that match alternate allele.
       Allele-specific counts are stored at the position of the SNP.

     --other_as_counts OTHER_AS_COUNT_H5_FILE [required]
       Path to HDF5 file to write counts of reads that match neither reference
       nor alternate allele. Allele-specific counts are stored at the position
       of the SNP.

     --read_counts READ_COUNT_H5_FILE [required]
       Path to HDF5 file to write counts of all reads, regardless of whether
       they overlap a SNP. Read counts are stored at the left-most position
       of the mapped read.
"""

import sys
import os
import gzip

import tables
import argparse
import numpy as np

import pysam

import chromosome
import chromstat

# codes used by pysam for aligned read CIGAR strings
BAM_CMATCH     = 0 # M
BAM_CINS       = 1 # I
BAM_CDEL       = 2 # D
BAM_CREF_SKIP  = 3 # N
BAM_CSOFT_CLIP = 4 # S
BAM_CHARD_CLIP = 5 # H
BAM_CPAD       = 6 # P
BAM_CEQUAL     = 7 # =
BAM_CDIFF      = 8 # X

BAM_CIGAR_DICT = {0 : "M",
                  1 : "I",
                  2 : "D",
                  3 : "N",
                  4 : "S",
                  5 : "H",
                  6 : "P",
                  7 : "=",
                  8 : "X"}

SNP_UNDEF = -1
MAX_UINT8_COUNT = 255
MAX_UINT16_COUNT = 65535

def create_carray(h5f, chrom, data_type):
    if data_type == "uint8":
        atom = tables.UInt8Atom(dflt=0)
    elif data_type == "uint16":
        atom = tables.UInt16Atom(dflt=0)
    else:
        raise NotImplementedError("unsupported datatype %s" % data_type)

    zlib_filter = tables.Filters(complevel=1, complib="zlib")

    # create CArray for this chromosome
    shape = [chrom.length]
    carray = h5f.createCArray(h5f.root, chrom.name,
                              atom, shape, filters=zlib_filter)

    return carray

def get_carray(h5f, chrom):
    return h5f.getNode("/%s" % chrom)

def is_indel(snp):
    if (len(snp['allele1']) != 1) or (len(snp['allele2'])) != 1:
        return True

def dump_read(f, read):
    cigar_str = " ".join(["%s:%d" % (BAM_CIGAR_DICT[c[0]], c[1])
                          for c in read.cigar])

    f.write("pos: %d\n"
            "aend: %d\n"
            "alen (len of aligned portion of read on genome): %d\n"
            "qstart: %d\n"
            "qend: %d\n"
            "qlen (len of aligned qry seq): %d\n"
            "rlen (read len): %d\n"
            "tlen (insert size): %d\n"
            "cigar: %s\n"
            "seq: %s\n"
            % (read.pos, read.aend, read.alen, read.qstart, read.qend,
               read.qlen, read.rlen, read.tlen, cigar_str, read.seq))

def get_sam_iter(samfile, chrom):
    try:
        sam_iter = samfile.fetch(reference=chrom.name,
                                 start=1, end=chrom.length)
    except ValueError as ve:
        sys.stderr.write("%s\n" % str(ve))
        # could not find chromosome, try stripping leading 'chr'
        # E.g. for drosophila, sometimes 'chr2L' is used but
        # othertimes just '2L' is used. Annoying!
        chrom_name = chrom.name.replace("chr", "")
        sys.stderr.write("WARNING: %s does not exist in BAM file, "
                         "trying %s instead\n" % (chrom.name, chrom_name))

        try:
            sam_iter = samfile.fetch(reference=chrom_name,
                                     start=1, end=chrom.length)
        except ValueError:
            sys.stderr.write("WARNING: %s does not exist in BAM file, "
                             "skipping chromosome\n" % chrom_name)
            sam_iter = iter([])

    return sam_iter

def choose_overlap_snp(read, snp_tab, snp_index_array, hap_tab, ind_idx):
    """Picks out a single SNP from those that the read overlaps.
    Returns a tuple containing 4 elements: [0] the index of the SNP in
    the SNP table, [1] the offset into the read sequence, [2] flag
    indicating whether the read was 'split' (i.e. was a spliced
    read), [3] flag indicating whether read overlaps known indel.
    If there are no overlapping SNPs or the read cannot be processed,
    (None, None, is_split, overlap_indel) is returned instead.
    """
    read_offsets = []
    snp_idx = []

    read_start_idx = 0
    genome_start_idx = read.pos

    n_match_segments = 0
    is_split = False
    overlap_indel = False

    for cig in read.cigar:
        op = cig[0]
        op_len = cig[1]

        if op == BAM_CMATCH:
            # this is a block of match/mismatch in read alignment
            read_end = read_start_idx + op_len
            genome_end = genome_start_idx + op_len

            # get offsets of any SNPs that this read overlaps
            idx = snp_index_array[genome_start_idx:genome_end]
            is_def = np.where(idx != SNP_UNDEF)[0]
            read_offsets.extend(read_start_idx + is_def)
            snp_idx.extend(idx[is_def])

            read_start_idx = read_end
            genome_start_idx = genome_end

            n_match_segments += 1
        elif op == BAM_CREF_SKIP:
            # spliced read, skip over this region of genome
            genome_start_idx += op_len
            is_split = True
        elif op == BAM_CSOFT_CLIP:
            # end of read is soft-clipped, which means it is
            # present in read, but not used in alignment
            read_start_idx += op_len
        elif op == BAM_CHARD_CLIP:
            # end of read is hard-clipped, so not present
            # in read and not used in alignment
            pass
        else:
            sys.stderr.write("skipping because contains CIGAR code %s "
                             " which is not currently implemented" %
                             BAM_CIGAR_DICT[op])

    # are any of the SNPs indels? If so, discard.
    for i in snp_idx:
        if is_indel(snp_tab[i]):
            overlap_indel = True
            return (None, None, is_split, overlap_indel)

    n_overlap_snps = len(read_offsets)
    if n_overlap_snps == 0:
        # no SNPs overlap this read
        return (None, None, is_split, overlap_indel)

    if hap_tab:
        # genotype info is provided by haplotype table
        # pull out subset of overlapping SNPs that are heterozygous
        # in this individual
        het_read_offsets = []
        het_snp_idx = []
        for (i, read_offset) in zip(snp_idx, read_offsets):
            haps = hap_tab[i, (ind_idx*2):(ind_idx*2 + 2)]

            if ind_idx*2 > hap_tab.shape[1]:
                raise ValueError("index of individual (%d) is >= number of "
                                 "individuals in haplotype_tab (%d). probably "
                                 "need to specify --population or use a different "
                                 "--samples_tab" % (ind_idx, hap_tab.shape[1]/2))

            if haps[0] != haps[1]:
                # this is a het
                het_read_offsets.append(read_offset)
                het_snp_idx.append(i)

        n_overlap_hets = len(het_read_offsets)

        if n_overlap_hets == 0:
            # none of the overlapping SNPs are hets
            return (None, None, is_split, overlap_indel)

        if n_overlap_hets == 1:
            # only one overlapping SNP is a het
            return (het_snp_idx[0], het_read_offsets[0], is_split, overlap_indel)

        # choose ONE overlapping HETEROZYGOUS SNP randomly to add counts to
        # we don't want to count same read multiple times
        r = np.random.randint(0, n_overlap_hets-1)
        return (het_snp_idx[r], het_read_offsets[r], is_split, overlap_indel)

    else:
        # We don't have haplotype tab, so we don't know which SNPs are
        # heterozygous in this individual. But we can still tell
        # whether read sequence matches reference or non-reference
        # allele. Choose ONE overlapping SNP randomly to add counts to
        if n_overlap_snps == 1:
            return (snp_idx[0], read_offsets[0], is_split, overlap_indel)
        else:
            r = np.random.randint(0, n_overlap_snps-1)
            return (snp_idx[r], read_offsets[r], is_split, overlap_indel)

def add_read_count(read, chrom, ref_array, alt_array, other_array,
                   read_count_array, snp_index_array, snp_tab, hap_tab,
                   warned_pos, max_count, ind_idx):

    # pysam positions start at 0
    start = read.pos+1
    end = read.aend

    if start < 1 or end > chrom.length:
        sys.stderr.write("WARNING: skipping read aligned past end of "
                         "chromosome. read: %d-%d, %s:1-%d\n" %
                         (start, end, chrom.name, chrom.length))
        return

    if read.qlen != read.rlen:
        sys.stderr.write("WARNING skipping read: handling of "
                         "partially mapped reads not implemented\n")
        return

    # look for SNPs that overlap mapped read position, and if there
    # are more than one, choose one at random
    snp_idx, read_offset, is_split, overlap_indel = \
      choose_overlap_snp(read, snp_tab, snp_index_array, hap_tab, ind_idx)

    if overlap_indel:
        return

    # store counts of reads at start position
    if read_count_array[start-1] < max_count:
        read_count_array[start-1] += 1
    else:
        if not start in warned_pos:
            sys.stderr.write("WARNING read count at position %d "
                             "exceeds max %d\n" % (start, max_count))
            warned_pos[start] = True

    if snp_idx is None:
        return

    snp = snp_tab[snp_idx]

    base = read.seq[read_offset]
    snp_pos = snp['pos']

    if base == snp['allele1']:
        # matches reference allele
        if ref_array[snp_pos-1] < max_count:
            ref_array[snp_pos-1] += 1
        elif not snp_pos in warned_pos:
            sys.stderr.write("WARNING ref allele count at position %d "
                             "exceeds max %d\n" % (snp_pos, max_count))
            warned_pos[snp_pos] = True
    elif base == snp['allele2']:
        # matches alternate allele
        if alt_array[snp_pos-1] < max_count:
            alt_array[snp_pos-1] += 1
        elif not snp_pos in warned_pos:
            sys.stderr.write("WARNING alt allele count at position %d "
                             "exceeds max %d\n" % (snp_pos, max_count))
            warned_pos[snp_pos] = True
    else:
        # matches neither
        if other_array[snp_pos-1] < max_count:
            other_array[snp_pos-1] += 1
        elif not snp_pos in warned_pos:
            sys.stderr.write("WARNING other allele count at position %d "
                             "exceeds max %d\n" % (snp_pos, max_count))

def parse_args():
    parser = argparse.ArgumentParser()

    parser.add_argument("--chrom",
                        help="Path to chromInfo.txt file (may be gzipped) "
                        "with list of chromosomes for the relevant genome "
                        "assembly. Each line in file should contain "
                        "tab-separated chromosome name and chromosome length "
                        "(in basepairs). chromInfo.txt files can be "
                        "downloaded from the UCSC genome browser. For "
                        "example, a chromInfo.txt.gz file for hg19 can "
                        "be downloaded from "
                        "http://hgdownload.soe.ucsc.edu/goldenPath/hg19/database/",
                        metavar="CHROM_TXT_FILE",
                        required=True)

    parser.add_argument("--snp_index",
                        help="Path to HDF5 file containing SNP index. The "
                        "SNP index is used to convert the genomic position "
                        "of a SNP to its corresponding row in the haplotype "
                        "and snp_tab HDF5 files.",
                        metavar="SNP_INDEX_H5_FILE",
                        required=True)

    parser.add_argument("--snp_tab",
                        help="Path to HDF5 file to read SNP information "
                        "from. Each row of SNP table contains SNP name "
                        "(rs_id), position, allele1, allele2.",
                        metavar="SNP_TABLE_H5_FILE",
                        required=True)

    parser.add_argument("--haplotype",
                        help=" Path to HDF5 file to read phased haplotypes "
                        "from. If supplied, when read overlaps multiple SNPs "
                        "counts are randomly assigned to ONE of the "
                        "overlapping HETEROZYGOUS SNPs; if not supplied "
                        "counts are randomly assigned to ONE of overlapping "
                        "SNPs (regardless of their genotype).",
                        metavar="HAPLOTYPE_H5_FILE",
                        default=None)

    parser.add_argument("--samples",
                        help="Path to text file containing a list of "
                        "individual identifiers. The ordering of individuals "
                        "must be consistent with the haplotype file. The "
                        "samples file is assumed to have one identifier per "
                        "line in the first column (other columns are "
                        "ignored).",
                        metavar="SAMPLES_TXT_FILE",
                        default=None)

    parser.add_argument("--individual",
                        help="Identifier for individual, used to determine "
                        "which SNPs are heterozygous. Must be provided "
                        "if --haplotype argument is provided and must "
                        "match one of the individuals in the file provided "
                        "with --samples argument.",
                        metavar="INDIVIDUAL",
                        default=None)

    parser.add_argument("--data_type",
                        help="Data type of counts stored in HDF5 files. "
                        "uint8 requires less disk space but has a "
                        "maximum value of 255."
                        "(default=uint8)", choices=("uint8", "uint16"),
                        default="uint8")

    parser.add_argument("--ref_as_counts",
                        help="Path to HDF5 file to write counts of reads "
                        "that match reference allele. Allele-specific counts "
                        "are stored at the position of the SNP."
                        "that match reference",
                        metavar="REF_AS_COUNT_H5_FILE",
                        required=True)

    parser.add_argument("--alt_as_counts",
                        help="Path to HDF5 file to write counts of reads "
                        "that match alternate allele. Allele-specific counts "
                        "are stored at the position of the SNP.",
                        metavar="ALT_AS_COUNT_H5_FILE",
                        required=True)

    parser.add_argument("--other_as_counts",
                        help="Path to HDF5 file to write counts of reads "
                        "that match neither reference nor alternate allele. "
                        "Allele-specific counts are stored at the position "
                        "of the SNP.",
                        metavar="OTHER_COUNT_H5_FILE",
                        required=True)

    parser.add_argument("--read_counts",
                       help="Path to HDF5 file to write counts of all "
                       "reads, regardless of whether they overlap a SNP. "
                       "Read counts are stored at the left-most position "
                       "of the mapped read.",
                       metavar="READ_COUNT_H5_FILE",
                       required=True)

    parser.add_argument("bam_filenames", action="store", nargs="+",
                        help="BAM file(s) to read mapped reads from. "
                        "BAMs must be sorted and indexed.")

    args = parser.parse_args()

    if args.haplotype and (args.individual is None or args.samples is None):
            parser.error("--indidivual and --samples arguments "
                         "must also be provided when --haplotype argument "
                         "is provided")

    return args

def lookup_individual_index(samples_file, ind_name, population=None):
    """Gets the index of individual that is used
    to lookup information in the genotype and haplotype tables"""
    f = open(samples_file)

    if population:
        p = population.lower()
    else:
        p = None

    idx = 0
    for line in f:
        if line.startswith("samples"):
            # header line
            continue

        words = line.rstrip().split()
        name = words[0].replace("NA", "")

        if len(words) > 1:
            pop = words[1].lower()
        else:
            pop = ""

        if len(words) > 2:
            group = words[2].lower()
        else:
            group = ""

        # if specified, only consider a single population or group
        if p and pop != p and group != p:
            continue

        if name == ind_name:
            f.close()
            return idx

        idx += 1

    raise ValueError("individual %s (with population=%s) "
                     "is not in samples file %s" %
                     (ind_name, population, samples_file))

def main():
    args = parse_args()

    snp_tab_h5 = tables.openFile(args.snp_tab, "r")
    snp_index_h5 = tables.openFile(args.snp_index, "r")

    if args.haplotype:
        hap_h5 = tables.openFile(args.haplotype, "r")
        ind_idx = lookup_individual_index(args.samples, args.individual)
    else:
        hap_h5 = None
        ind_idx = None

    ref_count_h5 = tables.openFile(args.ref_as_counts, "w")
    alt_count_h5 = tables.openFile(args.alt_as_counts, "w")
    other_count_h5 = tables.openFile(args.other_as_counts, "w")
    read_count_h5 = tables.openFile(args.read_counts, "w")

    output_h5 = [ref_count_h5, alt_count_h5, other_count_h5, read_count_h5]

    chrom_dict = {}

    # initialize every chromosome in output files
    chrom_list = chromosome.get_all_chromosomes(args.chrom)

    for chrom in chrom_list:
        for out_file in output_h5:
            create_carray(out_file, chrom, args.data_type)
        chrom_dict[chrom.name] = chrom

    count = 0
    dtype = None
    if args.data_type == "uint8":
        max_count = MAX_UINT8_COUNT
        dtype = np.uint8
    elif args.data_type == "uint16":
        max_count = MAX_UINT16_COUNT
        dtype = np.uint16
    else:
        raise NotImplementedError("unsupported datatype %s" % args.data_type)

    for chrom in chrom_list:
        sys.stderr.write("%s\n" % chrom.name)

        warned_pos = {}

        # fetch SNP info for this chromosome
        if chrom.name not in snp_tab_h5.root:
            # no SNPs for this chromosome
            continue

        sys.stderr.write("fetching SNPs\n")

        snp_tab = snp_tab_h5.getNode("/%s" % chrom.name)
        snp_index_array = snp_index_h5.getNode("/%s" % chrom.name)[:]
        if hap_h5:
            hap_tab = hap_h5.getNode("/%s" % chrom.name)
        else:
            hap_tab = None

        # initialize count arrays for this chromosome to 0
        ref_carray = get_carray(ref_count_h5, chrom)
        alt_carray = get_carray(alt_count_h5, chrom)
        other_carray = get_carray(other_count_h5, chrom)
        read_count_carray = get_carray(read_count_h5, chrom)

        ref_array = np.zeros(chrom.length, dtype)
        alt_array = np.zeros(chrom.length, dtype)
        other_array = np.zeros(chrom.length, dtype)
        read_count_array = np.zeros(chrom.length, dtype)

        # loop over all BAM files, pulling out reads
        # for this chromosome
        for bam_filename in args.bam_filenames:
            sys.stderr.write("reading from file %s\n" % bam_filename)

            samfile = pysam.Samfile(bam_filename, "rb")

            for read in get_sam_iter(samfile, chrom):
                count += 1
                if count == 10000:
                    sys.stderr.write(".")
                    count = 0

                add_read_count(read, chrom, ref_array, alt_array,
                               other_array, read_count_array,
                               snp_index_array, snp_tab, hap_tab,
                               warned_pos, max_count, ind_idx)

            # store results for this chromosome
            ref_carray[:] = ref_array
            alt_carray[:] = alt_array
            other_carray[:] = other_array
            read_count_carray[:] = read_count_array
            sys.stderr.write("\n")

            samfile.close()

    # set track statistics and close HDF5 files

    sys.stderr.write("setting statistics for each chromosome\n")
    for h5f in output_h5:
        chromstat.set_stats(h5f, chrom_list)
        h5f.close()

    snp_tab_h5.close()
    snp_index_h5.close()
    if hap_h5:
        hap_h5.close()

    sys.stderr.write("done\n")

main()


text: require File.dirname(__FILE__) + "/lib/validate_url.rb"


text:
package tabletserver

import (
	"fmt"
	"reflect"
	"strings"
	"testing"

	"github.com/youtube/vitess/go/sqltypes"

	querypb "github.com/youtube/vitess/go/vt/proto/query"
	vtrpcpb "github.com/youtube/vitess/go/vt/proto/vtrpc"
	"github.com/youtube/vitess/go/vt/vterrors"
	"github.com/youtube/vitess/go/vt/vttablet/tabletserver/schema"
)

func TestCodexBuildValuesList(t *testing.T) {
	table := createTable("Table",
		[]string{"pk1", "pk2", "col1"},
		[]querypb.Type{sqltypes.Int64, sqltypes.VarBinary, sqltypes.Int32},
		[]string{"pk1", "pk2"})

	// simple PK clause. e.g. where pk1 = 1
	bindVars := map[string]interface{}{}
	pk1Val, _ := sqltypes.BuildValue(1)
	pkValues := []interface{}{pk1Val}
	// want [[1]]
	want := [][]sqltypes.Value{{pk1Val}}
	got, _ := buildValueList(table, pkValues, bindVars)
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("got %v, want %v", got, want)
	}

	// simple PK clause with bindVars. e.g. where pk1 = :pk1
	bindVars["pk1"] = 1
	pkValues = []interface{}{":pk1"}
	// want [[1]]
	want = [][]sqltypes.Value{{pk1Val}}
	got, _ = buildValueList(table, pkValues, bindVars)
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("got %v, want %v", got, want)
	}

	// null value
	bindVars["pk1"] = nil
	pkValues = []interface{}{":pk1"}
	// want [[1]]
	want = [][]sqltypes.Value{{{}}}
	got, _ = buildValueList(table, pkValues, bindVars)
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("got %v, want %v", got, want)
	}

	// invalid value
	bindVars["pk1"] = struct{}{}
	pkValues = []interface{}{":pk1"}
	wantErr := "unexpected type struct {}: {}"

	got, err := buildValueList(table, pkValues, bindVars)

	if err == nil || !strings.Contains(err.Error(), wantErr) {
		t.Fatalf("got %v, want %v", err, wantErr)
	}

	// type mismatch int
	bindVars["pk1"] = "str"
	pkValues = []interface{}{":pk1"}
	wantErr = "strconv.ParseInt"

	got, err = buildValueList(table, pkValues, bindVars)
	if err == nil || !strings.Contains(err.Error(), wantErr) {
		t.Fatalf("got %v, want %v", err, wantErr)
	}

	// type mismatch binary
	bindVars["pk1"] = 1
	bindVars["pk2"] = 1
	pkValues = []interface{}{":pk1", ":pk2"}
	wantErr = "type mismatch, expecting string type for 1"

	got, err = buildValueList(table, pkValues, bindVars)
	if err == nil || !strings.Contains(err.Error(), wantErr) {
		t.Fatalf("got %v, want %v", err, wantErr)
	}

	// composite PK clause. e.g. where pk1 = 1 and pk2 = "abc"
	pk2Val, _ := sqltypes.BuildValue("abc")
	pkValues = []interface{}{pk1Val, pk2Val}
	// want [[1 abc]]
	want = [][]sqltypes.Value{{pk1Val, pk2Val}}
	got, _ = buildValueList(table, pkValues, bindVars)
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("got %v, want %v", got, want)
	}

	// multi row composite PK insert
	// e.g. insert into Table(pk1,pk2) values (1, "abc"), (2, "xyz")
	pk1Val2, _ := sqltypes.BuildValue(2)
	pk2Val2, _ := sqltypes.BuildValue("xyz")
	pkValues = []interface{}{
		[]interface{}{pk1Val, pk1Val2},
		[]interface{}{pk2Val, pk2Val2},
	}
	// want [[1 abc][2 xyz]]
	want = [][]sqltypes.Value{
		{pk1Val, pk2Val},
		{pk1Val2, pk2Val2}}
	got, _ = buildValueList(table, pkValues, bindVars)
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("got %v, want %v", got, want)
	}

	// composite PK IN clause
	// e.g. where pk1 = 1 and pk2 IN ("abc", "xyz")
	pkValues = []interface{}{
		pk1Val,
		[]interface{}{pk2Val, pk2Val2},
	}
	// want [[1 abc][1 xyz]]
	want = [][]sqltypes.Value{
		{pk1Val, pk2Val},
		{pk1Val, pk2Val2},
	}

	got, _ = buildValueList(table, pkValues, bindVars)
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("got %v, want %v", got, want)
	}

	// list arg two values
	// e.g. where pk1 = 1 and pk2 IN ::list
	bindVars = map[string]interface{}{
		"list": []interface{}{
			"abc",
			"xyz",
		},
	}
	pkValues = []interface{}{
		pk1Val,
		"::list",
	}
	// want [[1 abc][1 xyz]]
	want = [][]sqltypes.Value{
		{pk1Val, pk2Val},
		{pk1Val, pk2Val2},
	}
	got, _ = buildValueList(table, pkValues, bindVars)
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("got %v, want %v", got, want)
	}

	// list arg two values, using *querypb.BindVariable of type TUPLE
	// e.g. where pk1 = 1 and pk2 IN ::list
	bindVars = map[string]interface{}{
		"list": &querypb.BindVariable{
			Type: querypb.Type_TUPLE,
			Values: []*querypb.Value{
				{
					Type:  querypb.Type_VARBINARY,
					Value: []byte("abc"),
				},
				{
					Type:  querypb.Type_VARBINARY,
					Value: []byte("xyz"),
				},
			},
		},
	}
	pkValues = []interface{}{
		pk1Val,
		"::list",
	}
	// want [[1 abc][1 xyz]]
	want = [][]sqltypes.Value{
		{pk1Val, pk2Val},
		{pk1Val, pk2Val2},
	}
	got, err = buildValueList(table, pkValues, bindVars)
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("got %v / %v, want %v", got, err, want)
	}

	// list arg one value
	// e.g. where pk1 = 1 and pk2 IN ::list
	bindVars = map[string]interface{}{
		"list": []interface{}{
			"abc",
		},
	}
	pkValues = []interface{}{
		pk1Val,
		"::list",
	}
	// want [[1 abc][1 xyz]]
	want = [][]sqltypes.Value{
		{pk1Val, pk2Val},
	}
	got, _ = buildValueList(table, pkValues, bindVars)
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("got %v, want %v", got, want)
	}

	// list arg empty list
	bindVars = map[string]interface{}{
		"list": []interface{}{},
	}
	pkValues = []interface{}{
		pk1Val,
		"::list",
	}
	wantErr = "empty list supplied for list"
	got, err = buildValueList(table, pkValues, bindVars)
	if err == nil || !strings.Contains(err.Error(), wantErr) {
		t.Fatalf("got %v, want %v", err, wantErr)
	}

	// list arg for non-list
	bindVars = map[string]interface{}{
		"list": []interface{}{},
	}
	pkValues = []interface{}{
		pk1Val,
		":list",
	}
	wantErr = "unexpected arg type []interface {} for key list"
	got, err = buildValueList(table, pkValues, bindVars)
	if err == nil || !strings.Contains(err.Error(), wantErr) {
		t.Fatalf("got %v, want %v", err, wantErr)
	}
}

func TestCodexResolvePKValues(t *testing.T) {
	table := createTable("Table",
		[]string{"pk1", "pk2", "col1"},
		[]querypb.Type{sqltypes.Int64, sqltypes.VarBinary, sqltypes.Int32},
		[]string{"pk1", "pk2"})
	key := "var"
	bindVariables := make(map[string]interface{})
	bindVariables[key] = "1"

	pkValues := make([]interface{}, 0, 10)
	pkValues = append(pkValues, []interface{}{":" + key})
	// resolvePKValues should succeed for strings that can be converted to int.
	v, _, err := resolvePKValues(table, pkValues, bindVariables)
	if err != nil {
		t.Error(err)
	}
	wantV := []interface{}{[]sqltypes.Value{sqltypes.MakeTrusted(sqltypes.Int64, []byte("1"))}}
	if !reflect.DeepEqual(v, wantV) {
		t.Errorf("resolvePKValues: %#v, want %#v", v, wantV)
	}
	// resolvePKValues should fail because of conversion error.
	pkValues = make([]interface{}, 0, 10)
	pkValues = append(pkValues, sqltypes.MakeString([]byte("type_mismatch")))
	_, _, err = resolvePKValues(table, pkValues, nil)
	if code := vterrors.Code(err); code != vtrpcpb.Code_INVALID_ARGUMENT {
		t.Errorf("resolvePKValues: %v, want %v", code, vtrpcpb.Code_INVALID_ARGUMENT)
	}
	// pkValues with different length
	bindVariables = make(map[string]interface{})
	bindVariables[key] = 1
	key2 := "var2"
	key3 := "var3"
	bindVariables[key2] = "2"
	bindVariables[key3] = "3"
	pkValues = make([]interface{}, 0, 10)
	pkValues = append(pkValues, []interface{}{":" + key})
	pkValues = append(pkValues, []interface{}{":" + key2, ":" + key3})
	_, _, err = resolvePKValues(table, pkValues, bindVariables)
	if code := vterrors.Code(err); code != vtrpcpb.Code_INVALID_ARGUMENT {
		t.Errorf("resolvePKValues: %v, want %v", code, vtrpcpb.Code_INVALID_ARGUMENT)
	}
}

func TestCodexResolveListArg(t *testing.T) {
	testUtils := newTestUtils()
	table := createTable("Table",
		[]string{"pk1", "pk2", "col1"},
		[]querypb.Type{sqltypes.Int64, sqltypes.VarBinary, sqltypes.Int32},
		[]string{"pk1", "pk2"})

	key := "var"
	bindVariables := make(map[string]interface{})
	bindVariables[key] = []interface{}{fmt.Errorf("error is not supported")}

	_, err := resolveListArg(table.GetPKColumn(0), "::"+key, bindVariables)
	if code := vterrors.Code(err); code != vtrpcpb.Code_INVALID_ARGUMENT {
		t.Errorf("resolvePKValues: %v, want %v", code, vtrpcpb.Code_INVALID_ARGUMENT)
	}

	// This should successfully convert.
	bindVariables[key] = []interface{}{"1"}
	v, err := resolveListArg(table.GetPKColumn(0), "::"+key, bindVariables)
	if err != nil {
		t.Error(err)
	}
	wantV := []sqltypes.Value{sqltypes.MakeTrusted(sqltypes.Int64, []byte("1"))}
	if !reflect.DeepEqual(v, wantV) {
		t.Errorf("resolvePKValues: %#v, want %#v", v, wantV)
	}

	bindVariables[key] = []interface{}{10}
	result, err := resolveListArg(table.GetPKColumn(0), "::"+key, bindVariables)
	if err != nil {
		t.Fatalf("should not get an error, but got error: %v", err)
	}
	testUtils.checkEqual(t, []sqltypes.Value{sqltypes.MakeTrusted(sqltypes.Int64, []byte("10"))}, result)
}

func TestResolveNumber(t *testing.T) {
	testcases := []struct {
		v      interface{}
		bv     map[string]interface{}
		out    int64
		outErr string
	}{{
		v: ":a",
		bv: map[string]interface{}{
			"a": 10,
		},
		out: int64(10),
	}, {
		v: "::a",
		bv: map[string]interface{}{
			"a": []interface{}{10},
		},
		outErr: "unexpected type []interface {}: [10]",
	}, {
		v:      ":a",
		outErr: "missing bind var a",
	}, {
		v:      make(chan int),
		outErr: "unexpected type chan int",
	}, {
		v:   int64(1),
		out: int64(1),
	}, {
		v:      1.2,
		outErr: "strconv.ParseInt",
	}}
	for _, tc := range testcases {
		got, err := resolveNumber(tc.v, tc.bv)
		if err != nil {
			if !strings.Contains(err.Error(), tc.outErr) {
				t.Errorf("resolveNumber(%#v, %v): %v, must contain %s", tc.v, tc.bv, err, tc.outErr)
			}
			continue
		}
		if got != tc.out {
			t.Errorf("resolveNumber(%#v, %v): %d, want %d", tc.v, tc.bv, got, tc.out)
		}
	}
}

func TestCodexBuildSecondaryList(t *testing.T) {
	pk1 := "pk1"
	pk2 := "pk2"
	table := createTable("Table",
		[]string{"pk1", "pk2", "col1"},
		[]querypb.Type{sqltypes.Int64, sqltypes.VarBinary, sqltypes.Int32},
		[]string{pk1, pk2})

	// set pk2 = 'xyz' where pk1=1 and pk2 = 'abc'
	bindVars := map[string]interface{}{}
	pk1Val, _ := sqltypes.BuildValue(1)
	pk2Val, _ := sqltypes.BuildValue("abc")
	pkValues := []interface{}{pk1Val, pk2Val}
	pkList, _ := buildValueList(table, pkValues, bindVars)
	pk2SecVal, _ := sqltypes.BuildValue("xyz")
	secondaryPKValues := []interface{}{nil, pk2SecVal}
	// want [[1 xyz]]
	want := [][]sqltypes.Value{
		{pk1Val, pk2SecVal}}
	got, _ := buildSecondaryList(table, pkList, secondaryPKValues, bindVars)
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("case 1 failed, got %v, want %v", got, want)
	}

	secondaryPKValues = []interface{}{"invalid_type", 1}
	_, err := buildSecondaryList(table, pkList, secondaryPKValues, bindVars)
	if err == nil {
		t.Fatalf("should get an error, column 0 is int type, but secondary list provides a string")
	}
}

func TestCodexBuildStreamComment(t *testing.T) {
	pk1 := "pk1"
	pk2 := "pk2"
	table := createTable("Table",
		[]string{"pk1", "pk2", "col1"},
		[]querypb.Type{sqltypes.Int64, sqltypes.VarBinary, sqltypes.Int32},
		[]string{pk1, pk2})

	// set pk2 = 'xyz' where pk1=1 and pk2 = 'abc'
	bindVars := map[string]interface{}{}
	pk1Val, _ := sqltypes.BuildValue(1)
	pk2Val, _ := sqltypes.BuildValue("abc")
	pkValues := []interface{}{pk1Val, pk2Val}
	pkList, _ := buildValueList(table, pkValues, bindVars)
	pk2SecVal, _ := sqltypes.BuildValue("xyz")
	secondaryPKValues := []interface{}{nil, pk2SecVal}
	secondaryList, _ := buildSecondaryList(table, pkList, secondaryPKValues, bindVars)
	want := []byte(" /* _stream `Table` (pk1 pk2 ) (1 'YWJj' ) (1 'eHl6' ); */")
	got := buildStreamComment(table, pkList, secondaryList)
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("case 1 failed, got\n%s, want\n%s", got, want)
	}
}

func TestCodexValidateRow(t *testing.T) {
	table := createTable("Table",
		[]string{"pk1", "pk2", "col1"},
		[]querypb.Type{sqltypes.Int64, sqltypes.VarBinary, sqltypes.Int32},
		[]string{"pk1", "pk2"})
	// #columns and #rows do not match
	err := validateRow(table, []int{1}, []sqltypes.Value{})
	if code := vterrors.Code(err); code != vtrpcpb.Code_INVALID_ARGUMENT {
		t.Errorf("validateRow: %v, want %v", code, vtrpcpb.Code_INVALID_ARGUMENT)
	}
	// column 0 is int type but row is in string type
	err = validateRow(table, []int{0}, []sqltypes.Value{sqltypes.MakeString([]byte("str"))})
	if code := vterrors.Code(err); code != vtrpcpb.Code_INVALID_ARGUMENT {
		t.Errorf("validateRow: %v, want %v", code, vtrpcpb.Code_INVALID_ARGUMENT)
	}
}

func TestCodexApplyFilterWithPKDefaults(t *testing.T) {
	testUtils := newTestUtils()
	table := createTable("Table",
		[]string{"pk1", "pk2", "col1"},
		[]querypb.Type{sqltypes.Int64, sqltypes.VarBinary, sqltypes.Int32},
		[]string{"pk1", "pk2"})
	output := applyFilterWithPKDefaults(table, []int{-1}, []sqltypes.Value{})
	if len(output) != 1 {
		t.Fatalf("expect to only one output but got: %v", output)
	}
	val, err := output[0].ParseInt64()
	if err != nil {
		t.Fatalf("should not get an error, but got err: %v", err)
	}
	testUtils.checkEqual(t, int64(0), val)
}

func TestCodexUnicoded(t *testing.T) {
	testUtils := newTestUtils()
	in := "test"
	out := unicoded(in)
	testUtils.checkEqual(t, in, out)
	in = "tes\xFFFDt"
	out = unicoded(in)
	testUtils.checkEqual(t, "tes", out)
}

func createTable(name string, colNames []string, colTypes []querypb.Type, pKeys []string) *schema.Table {
	table := schema.NewTable(name)
	for i, colName := range colNames {
		colType := colTypes[i]
		defaultVal := sqltypes.Value{}
		if sqltypes.IsIntegral(colType) {
			defaultVal = sqltypes.MakeTrusted(sqltypes.Int64, []byte("0"))
		} else if colType == sqltypes.VarBinary {
			defaultVal = sqltypes.MakeString([]byte(""))
		}
		table.AddColumn(colName, colType, defaultVal, "")
	}
	setPK(table, pKeys)
	return table
}

func setPK(ta *schema.Table, colnames []string) error {
	if len(ta.Indexes) != 0 {
		panic("setPK must be called before adding other indexes")
	}
	pkIndex := ta.AddIndex("PRIMARY")
	for _, colname := range colnames {
		pkIndex.AddColumn(colname, 1)
	}
	ta.Done()
	return nil
}


text:
<?php

/**
 * Elgg invite form contents
 *
 * @package ElggInviteFriends
 */

$site = elgg_get_site_entity();
$default_message = elgg_echo('invitefriends:message:default', array($site->name));

$introduction = elgg_echo('invitefriends:introduction');

$emails_label = elgg_echo('invitefriends:emails');
$message_label = elgg_echo('invitefriends:message');

$emails = elgg_get_sticky_value('invitefriends', 'emails');
$message = elgg_get_sticky_value('invitefriends', 'emailmessage', $default_message);

$emails_textarea = elgg_view('input/plaintext', array(
	'id' => 'invitefriends-emails',
	'name' => 'emails',
	'value' => $emails,
	'rows' => 4,
		));
$message_textarea = elgg_view('input/plaintext', array(
	'id' => 'invitefriends-emailmessage',
	'name' => 'emailmessage',
	'value' => $message,
		));

$action_button = elgg_view('input/submit', array('value' => elgg_echo('send')));

echo <<< HTML

$introduction

	$emails_label
	$emails_textarea

	$message_label
	$message_textarea

	$action_button

HTML;


text:
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

<meta name="viewport" content="width=device-width, initial-scale=1.0">

<link rel="stylesheet" href="https://storage.googleapis.com/code.getmdl.io/1.0.2/material.orange-blue.min.css" />

<link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons">

<link rel="shortcut icon" href="img/favicon.ico" type="image/x-icon">

Client

                <input type="text" id="client" placeholder="Watch id">

Name - Vivo active

Position - 52.526768 | 13.393261

            Change

            watch

Hazard

                Please select the hazard place by clicking on the map:

left mouse click - minor hazard

right mouse click - major hazard

                Major |
                52.526768 |
                13.393261

            Report

            report_problem


text:
#!/usr/bin/env node

'use strict';

var counter = 0;
var delay = +process.env.DELAY || 5000;

console.log('Using ' + delay + 'ms delay...');

function ping() {
  console.log(process.pid, counter++);
  setTimeout(ping, delay);
}

ping();


text:
new vis.DataSet();
new vis.DataSet({});
new vis.DataSet([]);
new vis.DataSet([], {});

// Test Network constructor
new vis.Network(new HTMLDivElement(), { nodes: new vis.DataSet(), edges: new vis.DataSet() }, {});

//
// Test code sample from http://visjs.org/docs/data/dataset.html#Example
//

interface TestData {
  id: number;
  text?: string;
  date?: any;
  group?: number;
  balance?: number;
  first?: boolean;
}

// create a DataSet
let options = {};
let data = new vis.DataSet<TestData>(options);

// add items
// note that the data items can contain different properties and data formats
data.add([
  { id: 1, text: 'item 1', date: new Date(2013, 6, 20), group: 1, first: true },
  { id: 2, text: 'item 2', date: '2013-06-23', group: 2 },
  { id: 3, text: 'item 3', date: '2013-06-25', group: 2 },
  { id: 4, text: 'item 4' }
]);

// subscribe to any change in the DataSet
data.on('*', (event, properties, senderId) => {
  console.log('event', event, properties);
});

// update an existing item
data.update({ id: 2, group: 1 });

// remove an item
data.remove(4);

// get all ids
const ids = data.getIds();
console.log('ids', ids);

// get a specific item
const item1 = data.get(1);
console.log('item1', item1);

// retrieve a filtered subset of the data
let items = data.get({
  filter: (item) => {
    return item.group === 1;
  }
});
console.log('filtered items', items);

// retrieve formatted items
items = data.get({
  fields: ['id', 'date'],
  type: {
    date: 'ISODate'
  }
});
console.log('formatted items', items);

//
// Test code sample from http://visjs.org/docs/data/dataset.html#Subscriptions
//

// create a DataSet
data = new vis.DataSet<TestData>();

// subscribe to any change in the DataSet
data.on('*', (event, properties, senderId) => {
  console.log('event:', event, 'properties:', properties, 'senderId:', senderId);
});

// add an item
data.add({ id: 1, text: 'item 1' });              // triggers an 'add' event
data.update({ id: 1, text: 'item 1 (updated)' }); // triggers an 'update' event
data.remove(1);                                 // triggers an 'remove' event

//
// Test code sample from http://visjs.org/docs/data/dataset.html#Data_Manipulation
//

// create a DataSet
data = new vis.DataSet<TestData>();

// add items
data.add([
  { id: 1, text: 'item 1' },
  { id: 2, text: 'item 2' },
  { id: 3, text: 'item 3' }
]);

// update an item
data.update({ id: 2, text: 'item 2 (updated)' });

// remove an item
data.remove(3);

//
// Test code sample from http://visjs.org/docs/data/dataset.html#Data_Selection
//

// create a DataSet
data = new vis.DataSet<TestData>();
data.add([
  { id: 1, text: 'item 1', date: '2013-06-20', group: 1, first: true },
  { id: 2, text: 'item 2', date: '2013-06-23', group: 2 },
  { id: 3, text: 'item 3', date: '2013-06-25', group: 2 },
  { id: 4, text: 'item 4' }
]);

// retrieve formatted items
items = data.get({
  fields: ['id', 'date', 'group'],    // output the specified fields only
  type: {
    date: 'Date',                   // convert the date fields to Date objects
    group: 'String'                 // convert the group fields to Strings
  }
});

const dataset = new vis.DataSet<TestData>();

// retrieve all items having a property group with value 2
const group2 = dataset.get({
  filter: (item) => {
    return (item.group === 2);
  }
});

// retrieve all items having a property balance with a value above zero
const positiveBalance = dataset.get({
  filter: (item) => {
    return item.balance !== undefined && item.balance > 0;
  }
});

//
// Test code sample from Getting Started http://visjs.org/docs/network/
//

// create an array with nodes
const nodes = new vis.DataSet([
  { id: 1, label: 'Node 1' },
  { id: 2, label: 'Node 2' },
  { id: 3, label: 'Node 3' },
  { id: 4, label: 'Node 4' },
  { id: 5, label: 'Node 5' }
]);

// create an array with edges
const edges = new vis.DataSet([
  { from: 1, to: 3 },
  { from: 1, to: 2 },
  { from: 2, to: 4 },
  { from: 2, to: 5 }
]);

// create a network
const container = <HTMLElement> document.getElementById('mynetwork');

// provide the data in the vis format
const data2 = { nodes, edges };
options = {};

// initialize your network!
const network = new vis.Network(container, data2, options);

//
// Test code sample from http://visjs.org/docs/network/configure.html#
//

// these are all options in full.
const options2 = {
  configure: {
    enabled: true,
    filter: 'nodes,edges',
    container,
    showButton: true
  }
};

network.setOptions(options2);


text:
using System.Collections.Generic;
using System.Reflection;
using UnityEditor;
using UnityEngine;
using Vexe.Editor.GUIs;
using Vexe.Runtime.Helpers;
using Vexe.Runtime.Types;
using UnityObject = UnityEngine.Object;

namespace Vexe.Editor.Internal
{
	public class MembersCategory
	{
		private readonly EditorRecord _prefs;

		private readonly int _id;
		private CategoryDisplay _display;

		public readonly string FullPath;
		public readonly string Name;

		public List<MemberInfo> Members;
		public List<MembersCategory> NestedCategories;

		public int Spacing;
		public BaseGUI gui;
		public float DisplayOrder;
		public bool ForceExpand, HideHeader, IsExpanded, Indent, AlwaysHideHeader;

		public CategoryDisplay Display
		{
			get { return _display; }
			set
			{
				if (_display != value)
				{
					_display = value;
					for (int i = 0; i < NestedCategories.Count; i++)
						NestedCategories[i].Display = value;
				}
			}
		}

		public MembersCategory(string fullPath, float displayOrder, int id, EditorRecord prefs)
		{
			_prefs = prefs;
			this.DisplayOrder = displayOrder;
			this.FullPath = fullPath;
			this.Name = FullPath.Substring(FullPath.LastIndexOf('/') + 1);
			this._id = RuntimeHelper.CombineHashCodes(id, fullPath.GetHashCode());
			Indent = true;

			NestedCategories = new List<MembersCategory>();
			Members = new List<MemberInfo>();
		}

		public void RemoveEmptyNestedCategories()
		{
			for (int i = NestedCategories.Count - 1; i > -1; i--)
			{
				var nested = NestedCategories[i];
				if (nested.Members.Count == 0 && nested.NestedCategories.Count == 0)
					NestedCategories.RemoveAt(i);
				else nested.RemoveEmptyNestedCategories();
			}
		}

		private bool DoHeader()
		{
			bool foldout = false;
			var boxed = (_display & CategoryDisplay.BoxedHeaders) == CategoryDisplay.BoxedHeaders;
			using (gui.Horizontal(boxed ? GUIStyles.ToolbarButton : GUIStyle.none))
			{
				gui.Space(10f);
				foldout = gui.Foldout(Name, _prefs.ValueOrDefault(_id, true), Layout.Auto);
				_prefs[_id] = foldout;
			}

			return foldout;
		}

		public void Draw(UnityObject target)
		{
			if (Members.Count == 0 && NestedCategories.Count == 0)
				return;

			IsExpanded = AlwaysHideHeader || HideHeader || DoHeader();
			if (!(IsExpanded || ForceExpand))
				return;

			Indent = !(AlwaysHideHeader || HideHeader);

			gui.Space(1f);

			bool showGuiBox = (Display & CategoryDisplay.BoxedMembersArea) > 0;
			bool memberSplitter = (Display & CategoryDisplay.MemberSplitter) > 0;

			using (gui.Indent(showGuiBox ? EditorStyles.textArea : GUIStyle.none, Indent))
			{
				gui.Space(5f);
				for (int i = 0, imax = Members.Count; i < imax; i++)
				{
					var member = Members[i];

					var isVisible = ConditionalVisibility.IsVisible(member, target);
					if (!isVisible)
						continue;

					using (gui.Horizontal())
					{
						gui.Space(Spacing);
						using (gui.Vertical())
							gui.MemberField(member, target, target, _id, false);
					}

					if (memberSplitter && i != imax - 1)
						gui.Splitter();

					gui.Space(2f);
				}

				for (int i = 0, imax = NestedCategories.Count; i < imax; i++)
				{
					var cat = NestedCategories[i];

					cat.gui = this.gui;
					cat.HideHeader = this.HideHeader;
					cat.Display = _display;

					if ((_display & CategoryDisplay.CategorySplitter) != 0)
						gui.Splitter();

					using (gui.Horizontal())
					{
						if (IsExpanded)
							gui.Space(4f);

						using (gui.Vertical())
							cat.Draw(target);
					}
				}
			}
		}
	}
}


text:
import 'chrome://resources/cr_components/chromeos/cellular_setup/base_page.js';

import {flush} from 'chrome://resources/polymer/v3_0/polymer/polymer_bundled.min.js';

suite('CrComponentsBasePageTest', function() {
  let basePage;
  setup(function() {
    basePage = document.createElement('base-page');
    document.body.appendChild(basePage);
    flush();
  });

  test('Title is shown', function() {
    basePage.title = 'Base page titile';
    flush();
    const title = basePage.$$('#title');
    assertTrue(!!title);
  });

  test('Title is not shown', function() {
    const title = basePage.$$('#title');
    assertFalse(!!title);
  });

  test('Message icon is shown', function() {
    basePage.messageIcon = 'warning';
    flush();
    const messageIcon = basePage.$$('iron-icon');
    assertTrue(!!messageIcon);
    assertFalse(messageIcon.hidden);
  });

  test('Message icon is not shown', function() {
    const messageIcon = basePage.$$('iron-icon');
    assertTrue(!!messageIcon);
    assertTrue(messageIcon.hidden);
  });
});


text:
using Piranha.Extend.Fields;

namespace Piranha.Extend.Blocks;

/// <summary>
/// Single column quote block.
/// </summary>
[BlockType(Name = "Quote", Category = "Content", Icon = "fas fa-quote-right", Component = "quote-block")]
public class QuoteBlock : Block, ISearchable, ITranslatable
{
    /// <summary>
    /// Gets/sets the author
    /// </summary>
    public StringField Author { get; set; }

    /// <summary>
    /// Gets/sets the text body.
    /// </summary>
    public TextField Body { get; set; }

    /// <summary>
    /// Gets the title of the block when used in a block group.
    /// </summary>
    /// <returns>The title</returns>
    public override string GetTitle()
    {
        if (Body?.Value != null)
        {
            return Body.Value;
        }
        return "Empty";
    }

    /// <summary>
    /// Gets the content that should be indexed for searching.
    /// </summary>
    public string GetIndexedContent()
    {
        return !string.IsNullOrEmpty(Body.Value) ? Body.Value : "";
    }
}


text:
"""
SDS generic view
"""

__author__      = "Graham Klyne (GK@ACM.ORG)"
__copyright__   = "Copyright 2014, G. Klyne"
__license__     = "MIT (http://opensource.org/licenses/MIT)"

import os
import os.path
import json
import random
import uuid
import copy

# import httplib2

import logging
log = logging.getLogger(__name__)

from django.http                    import HttpResponse
from django.http                    import HttpResponseRedirect
from django.template                import RequestContext, loader
from django.views                   import generic
from django.views.decorators.csrf   import csrf_exempt
from django.core.urlresolvers       import resolve, reverse

from django.conf import settings

from utils.ContentNegotiationView   import ContentNegotiationView

from sds                            import message

LOGIN_URIS = None

class SdsGenericView(ContentNegotiationView):
    """
    Common base class for Sds views
    """

    def __init__(self):
        super(SdsGenericView, self).__init__()
        return

    def error(self, values):
        """
        Construct HTTP error response.

        This is an application-specific override of a method defined
        in ContentNegotiationView.
        """
        template = loader.get_template('sds_error.html')
        context  = RequestContext(self.request, values)
        return HttpResponse(template.render(context), status=values['status'], reason=values['reason'])

    def view_uri(self, viewname, **kwargs):
        """
        Return view URI given view name and any additional arguments
        """
        return reverse(viewname, kwargs=kwargs)

    def info_params(self, info_message, info_head=message.ACTION_COMPLETED):
        """
        Returns a URI query parameter string with details that are used to generate an
        information message.
        """
        return "?info_head=%s&info_message=%s"%(info_head, info_message)

    def redirect_info(self, viewuri, info_message=None, info_head=message.ACTION_COMPLETED):
        """
        Redirect to a specified view with an information/confirmation message for display

        (see templates/base_generic.html for display details)
        """
        redirect_uri = viewuri+self.info_params(info_message, info_head)
        return HttpResponseRedirect(redirect_uri)

    def error_params(self, error_message, error_head=message.INPUT_ERROR):
        """
        Returns a URI query parameter string with details that are used to generate an
        error message.
        """
        return "?error_head=%s&error_message=%s"%(error_head, error_message)

    def redirect_error(self, viewuri, error_message=None, error_head=message.INPUT_ERROR):
        """
        Redirect to a specified view with an error message for display

        (see templates/base_generic.html for display details)
        """
        redirect_uri = viewuri+self.error_params(error_head, error_message)
        return HttpResponseRedirect(redirect_uri)

    def check_value_supplied(self, val, msg, continuation_uri="", testfn=(lambda v: v)):
        """
        Test a supplied value is specified (not None) and passes a supplied test,
        returning a URI to display a supplied error message if the test fails.

        NOTE: this function works with the generic base template base_generic.html, which
        is assumed to provide an underlay for thne currently viewed page.

        val         value that is required to be not None and not empty or False
        msg         message to display if the value evaluated to False
        testfn      is a function to test the value (if not None).  If not specified,
                    the default test checks thatthe value does not evaluate as false
                    (e.g. is a non-empty string, list or collection).

        returns a URI string for use with HttpResponseRedirect to redisplay the
        current page with the supplied message, or None if the value passes the test.
        """
        redirect_uri = None
        if (val is None) or not testfn(val):
            redirect_uri = (
                self.get_request_path()+
                self.error_params(msg)
                ) + continuation_uri
        return redirect_uri

    # Authentication and authorization
    def authenticate(self):
        """
        Return None if required authentication is present, otherwise
        an appropriate login redirection response.

        self.credential is set to credential that can be used to access resource
        """
        # @@TODO: move logic to oauth2 app
        # Cache copy of URIs to use with OAuth2 login
        global LOGIN_URIS
        if LOGIN_URIS is None:
            LOGIN_URIS = (
                { "login_form_uri": self.view_uri('LoginUserView')
                , "login_post_uri": self.view_uri('LoginPostView')
                , "login_done_uri": self.view_uri('LoginDoneView')
                })
        # Initiate OAuth2 login sequence, if neded
        return oauth2.views.confirm_authentication(self,
            continuation_uri=self.get_request_uri(),
            **LOGIN_URIS
            )

    def authorize(self, scope):
        """
        Return None if user is authorized to perform the requested operation,
        otherwise appropriate 401 Authorization Required or 403 Forbidden response.
        May be called with or without an authenticated user.

        scope       indication of the operation  requested to be performed.
                    e.g. "VIEW", "CREATE", "UPDATE", "DELETE", ...

        @@TODO add resource parameter

        @@TODO proper authorization framework

        For now, require authentication for anything other than VIEW scope.
        """
        return None
        if scope != "VIEW":
            if not self.request.user.is_authenticated():
                return self.error(self.error401values())
        return None

    @ContentNegotiationView.accept_types(["text/html", "application/html", "*/*"])
    def render_html(self, resultdata, template_name):
        """
        Construct an HTML response based on supplied data and template name.

        Also contains logic to interpolate message values from the incoming URI,
        for error and confirmation message displays.  These additional message
        displays are commonly handled by the "base_generic.html" underlay template.
        """
        def uri_param_val(name, default):
            """
            Incorporate values from the incoming URI into the result data, if not already defined.
            """
            if name not in resultdata:
                resultdata[name] = self.request.GET.get(name, default)
            return
        uri_param_val("info_head",       message.ACTION_COMPLETED)
        uri_param_val("info_message",    None)
        uri_param_val("error_head",      message.INPUT_ERROR)
        uri_param_val("error_message",   None)
        resultdata["auth_create"]   = self.authorize("CREATE") is None
        resultdata["auth_update"]   = self.authorize("UPDATE") is None
        resultdata["auth_delete"]   = self.authorize("DELETE") is None
        template  = loader.get_template(template_name)
        context   = RequestContext(self.request, resultdata)
        log.debug("render_html - data: %r"%(resultdata))
        return HttpResponse(template.render(context))

    @ContentNegotiationView.accept_types(["text/csv"])
    def render_csv(self, resultdata, template_name):
        """
        Construct CSV response based on supplied data and template name.

        Render values gridhead and griddata from the supplied context
        """
        rendered_result = ",".join(resultdata["gridhead"])
        for row in resultdata["griddata"]:
            rendered_result += ",".join(row)
        return HttpResponse(rendered_result)

    @ContentNegotiationView.accept_types(["text/json"])
    def render_json(self, resultdata, template_name):
        """
        Construct JSON response based on supplied data and template name.
        """
        json_result = (
            { "filepath": resultdata["filepath"]
            , "filetype": resultdata["filetype"]
            , "gridhead": list(resultdata["gridhead"])
            , "griddata": [ list(r) for r in resultdata["griddata"] ]
            })
        return HttpResponse(json.dumps(json_result))

    # Default view methods return 405 Forbidden

    def get(self, request):
        return self.error(self.error405values())

    def head(self, request):
        return self.error(self.error405values())

    def put(self, request):
        return self.error(self.error405values())

    def post(self, request):
        return self.error(self.error405values())

    def delete(self, request):
        return self.error(self.error405values())

# End.