<|start_filename|>tests/robot_parser.h<|end_filename|>
/*
 * MIT License
 *
 * Alexandria.org
 *
 * Copyright (c) 2021 <NAME>, <<EMAIL>>, et al.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 * 
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 * 
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include "robots.h"

BOOST_AUTO_TEST_SUITE(robot_parser)

BOOST_AUTO_TEST_CASE(parse) {
	std::string robots_content = "Sitemap: https://www.omnible.se/sitemap.xml\n"
		"User-agent: AlexandriaBot\n"
		"Disallow: *\n"
		"User-agent: *   # all agents\n"
		"Disallow: /*crawl=no*\n"
		"Disallow: /basket/add*\n"
	;
	std::string user_agent = "AlexandriaBot";
	googlebot::RobotsMatcher matcher;
	std::string url = "/visit";
	bool allowed = matcher.OneAgentAllowedByRobots(robots_content, user_agent, url);
	BOOST_CHECK(!allowed);
}

BOOST_AUTO_TEST_CASE(parse2) {
	std::string robots_content = string("Sitemap: https://www.omnible.se/sitemap.xml\n"
		"User-agent: *\n"
		"Disallow: /visit\n"
		"User-agent: AlexandriaBot\n"
		"Disallow: /10126597891759986715\n");

	std::string user_agent = "AlexandriaBot";
	googlebot::RobotsMatcher matcher;
	{
		std::string url = "https://www.omnible.se/10126597891759986715";
		bool allowed = matcher.OneAgentAllowedByRobots(robots_content, user_agent, url);
		BOOST_CHECK(!allowed);
	}
	{
		std::string url = "https://www.omnible.se/1012659789175998671";
		bool allowed = matcher.OneAgentAllowedByRobots(robots_content, user_agent, url);
		BOOST_CHECK(allowed);
	}

}



BOOST_AUTO_TEST_SUITE_END()


<|start_filename|>src/utils/thread_pool.cpp<|end_filename|>
/*
 * MIT License
 *
 * Alexandria.org
 *
 * Copyright (c) 2021 <NAME>, <<EMAIL>>, et al.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include "thread_pool.hpp"
#include <iostream>
#include <thread>
#include <future>
#include <queue>

namespace utils {

	thread_pool::thread_pool(size_t num_threads) {
		for (size_t i = 0; i < num_threads; i++) {
			m_workers.emplace_back([this]() {
				this->handle_work();
			});
		}
	}
	thread_pool::~thread_pool() {
		run_all();
	}

	void thread_pool::enqueue(std::function<void()> &&fun) {

		if (m_stop) {
			throw std::runtime_error("enqueue on stopped thread_pool not allowed");
		}

		m_queue_lock.lock();
		m_queue.emplace(std::move(fun));
		m_queue_lock.unlock();

		m_condition.notify_one();
	}

	void thread_pool::run_all() {
		if (m_stop) return; // Already stopped..
		m_queue_lock.lock();
		m_stop = true;
		m_queue_lock.unlock();
		m_condition.notify_all();

		for (std::thread &thread : m_workers) {
			if (thread.joinable()) {
				thread.join();
			}
		}
	}

	void thread_pool::handle_work() {
		while (true) {

			std::function<void()> task;

			{
				std::unique_lock<std::mutex> lock(m_queue_lock);
				m_condition.wait(lock, [this] {
					return m_stop || !m_queue.empty();
				});
				if (m_stop && m_queue.empty()) return;
				task = std::move(m_queue.front());
				m_queue.pop();
			}

			task();
		}
	}
	
}


<|start_filename|>benchmark.cpp<|end_filename|>

#include <cstdlib>
#include <chrono>
#include <thread>
#include <cstring>
#include <fstream>
#include <iostream>
#include <algorithm>
#include <chrono>
#include <span>

using namespace std;

void *malloc_on_each(void *) {
	for (size_t i = 0; i < 200; i++) {
		size_t len = 50000000;
		char *buffer = new char[len];
		memset(buffer, 0, len);
		delete [] buffer;
	}
	return NULL;
}

void *malloc_once(void *) {
	size_t len = 50000000;
	char *buffer = new char[len];
	for (size_t i = 0; i < 200; i++) {
		memset(buffer, 0, len);
	}
	delete [] buffer;
	return NULL;
}

struct SStruct {
	size_t a;
	size_t b;
	size_t c;
};

void *openfile(void *) {
	//ifstream infile("/mnt/3/full_text/url_to_domain_main_index.fti", ios::in);
	ifstream infile("/root/url_to_domain_main_index.fti", ios::in);
	infile.seekg(0, ios::end);
	const size_t filesize = infile.tellg();

	SStruct *buffer = new SStruct[71838804];
	infile.read((char *)buffer, filesize);
	infile.close();
	delete buffer;

	return NULL;
}  

int main() {

	const size_t sz = 20000000;
	SStruct *buffer = new SStruct[sz];
	for (size_t i = 0; i < sz; i++) {
		buffer[i].a = rand();
	}

	span<SStruct> sp(buffer, sz);

	std::chrono::_V2::system_clock::time_point start_time = std::chrono::high_resolution_clock::now();
	sort(sp.begin(), sp.end(), [](const struct SStruct &a, const struct SStruct &b) { return a.a < b.a; });

	auto timer_elapsed = chrono::high_resolution_clock::now() - start_time;
	auto microseconds = chrono::duration_cast<std::chrono::microseconds>(timer_elapsed).count();

	cout << "sort took: " << microseconds/1000 << " milliseconds\n";

	delete buffer;

	return 0;

	const size_t num_threads = 30;
	pthread_t thread_ids[num_threads];
	for (size_t i = 0; i < num_threads; i++) {
		pthread_create(&thread_ids[i], NULL, openfile, NULL);
	}
	for (size_t i = 0; i < num_threads; i++) {
		pthread_join(thread_ids[i], NULL);
	}

	return 0;
}
