<file_sep>import { Fef } from './fef/fef';
const { JSDOM } = require('jsdom');

const processor = new Fef('../../data/dataPort.csv', 'csv', {
});

processor.setInputPreparation((item) => {
	const [address, statusCode, status, ...links] = Object.values(item);
	const validLinks = links.filter((link) => link);

	if (validLinks.length) {
		return {
			address,
			links: validLinks,
		};
	} else {
		return undefined;
	}
});

processor.setItemTransformation(({ address, links }) => {
	const localURLsOpeningInNewTabs = links.map((link) => {
		const fragment = JSDOM.fragment(link);
		const a = fragment.children[0];
		const href = a.href;
		const textContent = a.textContent;
		let hostname = '';
		try {
			hostname = new URL(href).hostname;
		} catch (error) {
			console.warn(`Invalid input on page: ${address} => ${href}` );
			hostname = `INVALID: ${href}`;
		}

		return {
			href,
			textContent,
			hostname
		};
	}).filter(({hostname}) => hostname.includes('port.ac.uk') || hostname.includes('INVALID'));

	return localURLsOpeningInNewTabs.map(({href, textContent, hostname}) => ({
		page: address,
		textContent,
		hostname,
		linksTo: href
	}));
});

processor.run('../../data/out/dataPort.csv');
<file_sep>import { csvToJSON, jsonToCSV } from './csv';
import listsToObjects from '../utils/listsToObjects';

/** Options for debugging and development
 * @typedef {Object} DebugOptions
 * @property {number} limit - Only use the first n data items
 */
/** Options for use in the browser
 * @typedef {Object} BrowserOptions
 * @property {HTMLElement} downloadLinkElem
 * @property {function()} displayDownloadLink
 */
/** Configure Fef
 * @typedef {Object} FefOptions
 * @property {?DebugOptions} debug
 * @property {?BrowserOptions} browser
 */

/**
 * @typedef {string} url
 * @typedef {('csv'|'json')} IOFormats
 * @typedef {string} JSONString
 */

/**
 * Supported mime types
 */
export const mimes = {
	csv: 'text/csv',
	json: 'application/json'
};

/**
 * Methods that take received data and convert to a usable format
 */
const extractionMethods = {
	/** Process a CSV file, return as objects
	 * @param  {string} fileData
	 * @returns {Object[]}
	 */
	csv: (fileData) => {
		const jsonData = csvToJSON(fileData);
		return listsToObjects(jsonData[0], jsonData.slice(1));
	},
	/** Process a JSON file
	 * @param  {JSONString} fileData - A JSON string
	 * @returns {Object[]}
	 */
	json: (fileData) => JSON.parse(fileData)
};

/**
 * Methods to prepare data for export
 */
const exportPrepMethods = {
	/** Convert to CSV format
	 * @param  {Object[]} dataToPrepareForExport
	 * @returns {string} Data as a string that can be saved
	 */
	csv: (dataToPrepareForExport) => jsonToCSV(dataToPrepareForExport),
	/**
	 * @param  {Object[]} dataToPrepareForExport
	 * @returns  {JSONString} JSON string
	 */
	json: (dataToPrepareForExport) => JSON.stringify(dataToPrepareForExport)
};

/** Convert file content with a given filetype to data
 * @param  {string} data
 * @param  {IOFormats} dataType
 * @param  {boolean} debugging
 * @param  {DebugOptions} debugOptions
 * @returns {Object[]}
 */
export const extractData = (data, dataType, debugging, debugOptions) => {
	let extractedData = extractionMethods[dataType](data);

	if (debugging && debugOptions && debugOptions.limit) {
		extractedData = extractedData.slice(0, debugOptions.limit);
	}

	return extractedData;
};

/** Convert data to given format for exporting
 * @param  {IOFormats} format
 * @param  {Object[]} data
 */
export const setData = (format, data) => {
	if (!format) {
		throw new Error('setData didn\'t receive a format');
	}
	return exportPrepMethods[format](data);
};
<file_sep>#!/bin/bash

rm -r dist
if command -v npm &> /dev/null
then
	npm run build && node dist/index.js
else
	/usr/local/bin/npm run build &&\
	/usr/local/bin/node dist/index.js
fi<file_sep>export const getFromBrowser = (elementId) =>
	new Promise((resolve, reject) => {
		console.log('Fetching the element with this id: ', elementId);

		const elem = document.querySelector(elementId);
		console.log('Found this element: ', elem);

		if (elem === null) {
			reject('This isn\'t an element?');
		}

		if (elem.type !== 'file') {
			reject('Not a file input element');
		}

		const reader = new FileReader();
		const file = document.getElementById('uploadForm').files[0];
		console.log(file);

		reader.readAsText(file);
		reader.onload = (loadEvent) => {
			const extractedData = loadEvent.target.result;
			console.log('Data extracted by FileReader: ', extractedData);
			resolve(extractedData);
		};
		reader.onerror = (err) => {
			reject('An error from the FileReader: ', err);
		};
	});

export const saveToDevice = (
	exportableData,
	fileName,
	mime,
	downloadLinkElem,
	displayDownloadLink
) => {
	const file = new File(exportableData.split('\n'), fileName, { type: mime });
	const url = URL.createObjectURL(file);

	downloadLinkElem.href = url;
	downloadLinkElem.download = fileName;

	displayDownloadLink();
};
<file_sep>// Using the functions defined by <NAME>
// Source https://dev.to/karataev/handling-a-lot-of-requests-in-javascript-with-promises-1kbb
// Code from Karataev, notes added by AndrewStanton94

/**
 * Promisify a function called over a list. All resolve concurrently
 * @param {string[]} items - The items to be passed to the function
 * @param {function} fn - The function to run for each argument
 * @returns {Promise} The setted promise.all
 */
function all(items, fn) {
	const promises = items.map((item) => fn(item));
	return Promise.all(promises);
}

/**
 * Promisify a function called over a list. Promises resolved sequentially.
 * @param {string[]} items - The items to be passed to the function
 * @param {function} fn - The function to run for each argument
 * @returns {Promise} The setted promise.all
 */
function series(items, fn) {
	let result = [];
	return items
		.reduce((acc, item) => {
			acc = acc.then(() => {
				return fn(item).then((res) => result.push(res));
			});
			return acc;
		}, Promise.resolve())
		.then(() => result);
}

/**
 * Break list into more manageable segments
 * @param {string[]} items - The array to break down
 * @param {number} [chunkSize]
 * @returns {string[][]} - A list of list chunks
 */
function splitToChunks(items, chunkSize = 50) {
	const result = [];
	for (let i = 0; i < items.length; i += chunkSize) {
		result.push(items.slice(i, i + chunkSize));
	}
	return result;
}

/**
 * Takes a list of items, breaks it into manageable chunks that can be ran in parallel
 * @param {string[]} items - The items to be passed to the function
 * @param {function} fn - The function to run for each argument
 * @param {number} [chunkSize]
 */
function chunks(items, fn, chunkSize = 50) {
	let result = [];
	const chunks = splitToChunks(items, chunkSize);
	return series(chunks, (chunk) => {
		return all(chunk, fn).then((res) => (result = result.concat(res)));
	}).then(() => result);
}

module.exports = {
	all,
	series,
	splitToChunks,
	chunks,
};
<file_sep>const fs = require('fs');
const { join } = require('path');

/** Create an absolute filepath from a relative one
 * @param  {string} relPath
 * @returns {string} Absolute file path
 */
const absFilePath = (relPath) => join(__dirname, relPath);

/** Get the contents of a file on the local machine
 * @param  {string} url
 * @returns {string} File content
 */
export const getLocalFile = (url) =>
	new Promise((resolve, reject) => {
		const absURL = absFilePath(url);
		console.log('Fetching from this path: ', absURL);

		fs.readFile(absURL, 'utf-8', (err, data) => {
			if (err) {
				console.error('Issue getting the file:');
				reject(err);
			}

			resolve(data);
		});
	});

/** Save data to file
 * @param  {string} exportableData
 * @param  {string} outputPath
 */
export const saveLocalFile = (exportableData, outputPath) => {
	fs.writeFile(absFilePath(outputPath), exportableData, (err) => {
		if (err) throw err;
		console.log('Saved!', outputPath);
	});
};

/** Convert to JSON and save file
 * @param  {string} relPath
 * @param  {string} data
 */
export const writeJSON = (relPath, data) =>
	saveLocalFile(JSON.stringify(data), relPath);
<file_sep>export const processData = async (fef) => {
	if (fef.prepareEachInput) {
		console.log(
			`There are ${fef.data.extractedData.length} items from the file`
		);
	}

	if (fef.inputPreparation) {
		console.log('Processing input');

		let preparedItems;
		/**
		 * Process input as individual items or as a single object
		 */
		if (fef.prepareEachInput) {
			preparedItems = fef.data.extractedData
				.map(fef.inputPreparation)
				.filter((item) => typeof item !== 'undefined');
		} else {
			const { items, processor } = await fef.inputPreparation(
				fef.data.extractedData,
				fef
			);
			preparedItems = items;
			fef = processor;
		}

		const itemsRemaining = preparedItems.length;
		fef.stats.dataToProcess = itemsRemaining;
		if (!itemsRemaining) {
			return Promise.reject('Filtering removed all items');
		}
		console.log(`${itemsRemaining} remain after input preparation`);

		fef.data.validData = preparedItems;
	} else {
		console.log('Won\'t filter');
		fef.data.validData = fef.data.extractedData;
	}

	const processedData = fef.data.validData.flatMap((data) =>
		fef.transformation(data, fef)
	);
	return Promise.all(processedData).then((results) => {
		fef.data.processed = results;

		fef.saveJSON('../../data/debug/processedData.json', results);

		return fef;
	});
};
<file_sep>const yeahNah = (condition, data) => {
	// console.log('Checking items in list ', data);

	const yeah = [];
	const nah = [];
	data.forEach((element) => {
		if (condition(element)) {
			yeah.push(element);
		} else {
			nah.push(element);
		}
	});
	return { yeah, nah };
};

module.exports = {
	yeahNah,
};
<file_sep>import { Fef } from './fef/fef';

import axios from 'axios';

import { chunks } from './utils/karataev';
import { yeahNah } from './utils/utils';

// console.log(Fef);

window.addEventListener('load', () => {
	const processor = new Fef('#uploadForm', 'csv', {
		platform: 'browser',
		debug: { limit: 80 },
		browser: {
			downloadLinkElem: document.getElementById('downloadLink'),
			// From vue app
			// displayDownloadLink: this.setReadyToDownload,
		},
	});
	processor.setInputPreparation((academic) =>
		academic['Name variant > Known as name-0'] &&
		academic['UUID-1'] &&
		academic['Organisations > Email addresses > Email-2']
			? academic
			: undefined
	);

	processor.setItemTransformation((academic) => {
		const baseURL = 'https://researchportal.port.ac.uk/portal/en/persons/';

		/** Format the name for the URL
		 *
		 * @param {string} name The original name
		 * @returns {string} URL compatible name
		 */
		const deduplicate = (list) =>
			[...new Set(list)].map((str) => str.trim());
		const normaliseAcademicName = (name) =>
			name.replace(/\s/g, '-').toLowerCase();

		const name = academic['Name variant > Known as name-0'];
		const uuid = academic['UUID-1'];
		const emailString =
			academic['Organisations > Email addresses > Email-2'];
		const email = deduplicate(emailString.toLowerCase().split(', '));

		const url = `${baseURL}${normaliseAcademicName(
			name
		)}(${uuid})/publications.html`;
		return { url, name, uuid, email };
	});

	processor.setPostProcessValidation(
		(academics) =>
			new Promise((resolve, reject) => {
				const instance = axios.create();
				let exportAcademics;

				const requestURL = (academic, instance) =>
					instance
						.head(academic.url)
						.then(({ status, statusText, request }) => {
							let { url } = academic;
							const { path } = request;

							if (
								url.replace(
									'https://researchportal.port.ac.uk',
									''
								) !== path
							) {
								status = 301;
								statusText = 'Has been redirected';
							}

							const linkData = {
								url,
								status,
								statusText,
								path,
							};
							academic.linkData = linkData;

							return academic;
						})
						.catch(({ message }) => ({
							academic,
							error: message,
						}));

				chunks(
					academics,
					(academic) => requestURL(academic, instance),
					25
				).then((output) => {
					const {
						yeah: foundAcademics = [],
						nah: missingAcademics = [],
					} = yeahNah(
						(academic) => typeof academic.error === 'undefined',
						output
					);

					console.log('Valid items: ', foundAcademics);
					console.log('Rejections: ', missingAcademics);

					exportAcademics = foundAcademics.map(
						({ name, url, email }) => {
							return {
								name,
								url,
								email,
							};
						}
					);
					resolve(exportAcademics);
				});
			})
	);

	processor.run('testOut.csv', 'csv');
});
<file_sep># For-each Files (fef)

A framework for file processing.

It should:

1. Read HTML | CSV | XLSX files
2. Use user given functions to process the data
3. Save it to a convenient format

This started by dogfooding a couple of tasks. I want to refactor it for wider use.
