text:
package org.apache.flink.client;

import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.JobSubmissionResult;
import org.apache.flink.api.common.accumulators.AccumulatorHelper;
import org.apache.flink.client.cli.CancelOptions;
import org.apache.flink.client.cli.CliArgsException;
import org.apache.flink.client.cli.CliFrontendParser;
import org.apache.flink.client.cli.CommandLineOptions;
import org.apache.flink.client.cli.CustomCommandLine;
import org.apache.flink.client.cli.DefaultCLI;
import org.apache.flink.client.cli.InfoOptions;
import org.apache.flink.client.cli.ListOptions;
import org.apache.flink.client.cli.ProgramOptions;
import org.apache.flink.client.cli.RunOptions;
import org.apache.flink.client.cli.SavepointOptions;
import org.apache.flink.client.cli.StopOptions;
import org.apache.flink.client.program.ClusterClient;
import org.apache.flink.client.program.PackagedProgram;
import org.apache.flink.client.program.ProgramInvocationException;
import org.apache.flink.client.program.ProgramMissingJobException;
import org.apache.flink.client.program.ProgramParametrizationException;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.GlobalConfiguration;
import org.apache.flink.configuration.IllegalConfigurationException;
import org.apache.flink.configuration.JobManagerOptions;
import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.optimizer.DataStatistics;
import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.costs.DefaultCostEstimator;
import org.apache.flink.optimizer.plan.FlinkPlan;
import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plan.StreamingPlan;
import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.client.JobStatusMessage;
import org.apache.flink.runtime.instance.ActorGateway;
import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.runtime.messages.JobManagerMessages;
import org.apache.flink.runtime.messages.JobManagerMessages.CancelJob;
import org.apache.flink.runtime.messages.JobManagerMessages.CancelJobWithSavepoint;
import org.apache.flink.runtime.messages.JobManagerMessages.CancellationFailure;
import org.apache.flink.runtime.messages.JobManagerMessages.CancellationSuccess;
import org.apache.flink.runtime.messages.JobManagerMessages.RunningJobsStatus;
import org.apache.flink.runtime.messages.JobManagerMessages.StopJob;
import org.apache.flink.runtime.messages.JobManagerMessages.StoppingFailure;
import org.apache.flink.runtime.messages.JobManagerMessages.TriggerSavepoint;
import org.apache.flink.runtime.messages.JobManagerMessages.TriggerSavepointSuccess;
import org.apache.flink.runtime.security.SecurityUtils;
import org.apache.flink.runtime.util.EnvironmentInformation;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.StringUtils;

import org.apache.commons.cli.CommandLine;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.net.InetSocketAddress;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;

import scala.Option;
import scala.concurrent.Await;
import scala.concurrent.Future;
import scala.concurrent.duration.FiniteDuration;

import static org.apache.flink.runtime.messages.JobManagerMessages.DisposeSavepoint;
import static org.apache.flink.runtime.messages.JobManagerMessages.DisposeSavepointFailure;
import static org.apache.flink.runtime.messages.JobManagerMessages.TriggerSavepointFailure;

/**
 * Implementation of a simple command line frontend for executing programs.
 */
public class CliFrontend {

	private static final Logger LOG = LoggerFactory.getLogger(CliFrontend.class);

	// actions
	private static final String ACTION_RUN = "run";
	private static final String ACTION_INFO = "info";
	private static final String ACTION_LIST = "list";
	private static final String ACTION_CANCEL = "cancel";
	private static final String ACTION_STOP = "stop";
	private static final String ACTION_SAVEPOINT = "savepoint";

	// config dir parameters
	private static final String CONFIG_DIRECTORY_FALLBACK_1 = "../conf";
	private static final String CONFIG_DIRECTORY_FALLBACK_2 = "conf";

	// --------------------------------------------------------------------------------------------

	private static final List<CustomCommandLine<?>> customCommandLines = new ArrayList<>(3);

	static {
		//	Command line interface of the YARN session, with a special initialization here
		//	to prefix all options with y/yarn.
		//	Tips: DefaultCLI must be added at last, because getActiveCustomCommandLine(..) will get the
		//	      active CustomCommandLine in order and DefaultCLI isActive always return true.
		final String flinkYarnSessionCLI = "org.apache.flink.yarn.cli.FlinkYarnSessionCli";
		final String flinkYarnCLI = "org.apache.flink.yarn.cli.FlinkYarnCLI";
		try {
			customCommandLines.add(loadCustomCommandLine(flinkYarnSessionCLI, "y", "yarn"));
		} catch (Exception e) {
			LOG.warn("Could not load CLI class {}.", flinkYarnSessionCLI, e);
		}

		try {
			customCommandLines.add(loadCustomCommandLine(flinkYarnCLI, "y", "yarn"));
		} catch (Exception e) {
			LOG.warn("Could not load CLI class {}.", flinkYarnCLI, e);
		}
		customCommandLines.add(new DefaultCLI());
	}

	// --------------------------------------------------------------------------------------------

	private final Configuration config;

	private final FiniteDuration clientTimeout;

	private final int defaultParallelism;

	/**
	 *
	 * @throws Exception Thrown if the configuration directory was not found, the configuration could not be loaded
	 */
	public CliFrontend() throws Exception {
		this(getConfigurationDirectoryFromEnv());
	}

	public CliFrontend(String configDir) throws Exception {

		// configure the config directory
		File configDirectory = new File(configDir);
		LOG.info("Using configuration directory " + configDirectory.getAbsolutePath());

		// load the configuration
		LOG.info("Trying to load configuration file");
		this.config = GlobalConfiguration.loadConfiguration(configDirectory.getAbsolutePath());

		try {
			FileSystem.setDefaultScheme(config);
		} catch (IOException e) {
			throw new Exception("Error while setting the default " +
				"filesystem scheme from configuration.", e);
		}

		this.clientTimeout = AkkaUtils.getClientTimeout(config);
		this.defaultParallelism = GlobalConfiguration.loadConfiguration().getInteger(
														ConfigConstants.DEFAULT_PARALLELISM_KEY,
														ConfigConstants.DEFAULT_PARALLELISM);
	}

	// --------------------------------------------------------------------------------------------
	//  Getter & Setter
	// --------------------------------------------------------------------------------------------

	/**
	 * Getter which returns a copy of the associated configuration.
	 *
	 * @return Copy of the associated configuration
	 */
	public Configuration getConfiguration() {
		Configuration copiedConfiguration = new Configuration();

		copiedConfiguration.addAll(config);

		return copiedConfiguration;
	}

	// --------------------------------------------------------------------------------------------
	//  Execute Actions
	// --------------------------------------------------------------------------------------------

	/**
	 * Executions the run action.
	 *
	 * @param args Command line arguments for the run action.
	 */
	protected int run(String[] args) {
		LOG.info("Running 'run' command.");

		RunOptions options;
		try {
			options = CliFrontendParser.parseRunCommand(args);
		}
		catch (CliArgsException e) {
			return handleArgException(e);
		}
		catch (Throwable t) {
			return handleError(t);
		}

		// evaluate help flag
		if (options.isPrintHelp()) {
			CliFrontendParser.printHelpForRun();
			return 0;
		}

		if (options.getJarFilePath() == null) {
			return handleArgException(new CliArgsException("The program JAR file was not specified."));
		}

		PackagedProgram program;
		try {
			LOG.info("Building program from JAR file");
			program = buildProgram(options);
		}
		catch (FileNotFoundException e) {
			return handleArgException(e);
		}
		catch (Throwable t) {
			return handleError(t);
		}

		ClusterClient client = null;
		try {

			client = createClient(options, program);
			client.setPrintStatusDuringExecution(options.getStdoutLogging());
			client.setDetached(options.getDetachedMode());
			LOG.debug("Client slots is set to {}", client.getMaxSlots());

			LOG.debug(options.getSavepointRestoreSettings().toString());

			int userParallelism = options.getParallelism();
			LOG.debug("User parallelism is set to {}", userParallelism);
			if (client.getMaxSlots() != -1 && userParallelism == -1) {
				logAndSysout("Using the parallelism provided by the remote cluster ("
					+ client.getMaxSlots() + "). "
					+ "To use another parallelism, set it at the ./bin/flink client.");
				userParallelism = client.getMaxSlots();
			} else if (ExecutionConfig.PARALLELISM_DEFAULT == userParallelism) {
				userParallelism = defaultParallelism;
			}

			return executeProgram(program, client, userParallelism);
		}
		catch (Throwable t) {
			return handleError(t);
		}
		finally {
			if (client != null) {
				try {
					client.shutdown();
				} catch (Exception e) {
					LOG.warn("Could not properly shut down the cluster client.", e);
				}
			}
			if (program != null) {
				program.deleteExtractedLibraries();
			}
		}
	}

	/**
	 * Executes the info action.
	 *
	 * @param args Command line arguments for the info action.
	 */
	protected int info(String[] args) {
		LOG.info("Running 'info' command.");

		// Parse command line options
		InfoOptions options;
		try {
			options = CliFrontendParser.parseInfoCommand(args);
		}
		catch (CliArgsException e) {
			return handleArgException(e);
		}
		catch (Throwable t) {
			return handleError(t);
		}

		// evaluate help flag
		if (options.isPrintHelp()) {
			CliFrontendParser.printHelpForInfo();
			return 0;
		}

		if (options.getJarFilePath() == null) {
			return handleArgException(new CliArgsException("The program JAR file was not specified."));
		}

		// -------- build the packaged program -------------

		PackagedProgram program;
		try {
			LOG.info("Building program from JAR file");
			program = buildProgram(options);
		}
		catch (Throwable t) {
			return handleError(t);
		}

		try {
			int parallelism = options.getParallelism();
			if (ExecutionConfig.PARALLELISM_DEFAULT == parallelism) {
				parallelism = defaultParallelism;
			}

			LOG.info("Creating program plan dump");

			Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config);
			FlinkPlan flinkPlan = ClusterClient.getOptimizedPlan(compiler, program, parallelism);

			String jsonPlan = null;
			if (flinkPlan instanceof OptimizedPlan) {
				jsonPlan = new PlanJSONDumpGenerator().getOptimizerPlanAsJSON((OptimizedPlan) flinkPlan);
			} else if (flinkPlan instanceof StreamingPlan) {
				jsonPlan = ((StreamingPlan) flinkPlan).getStreamingPlanAsJSON();
			}

			if (jsonPlan != null) {
				System.out.println("----------------------- Execution Plan -----------------------");
				System.out.println(jsonPlan);
				System.out.println("--------------------------------------------------------------");
			}
			else {
				System.out.println("JSON plan could not be generated.");
			}

			String description = program.getDescription();
			if (description != null) {
				System.out.println();
				System.out.println(description);
			}
			else {
				System.out.println();
				System.out.println("No description provided.");
			}
			return 0;
		}
		catch (Throwable t) {
			return handleError(t);
		}
		finally {
			program.deleteExtractedLibraries();
		}
	}

	/**
	 * Executes the list action.
	 *
	 * @param args Command line arguments for the list action.
	 */
	protected int list(String[] args) {
		LOG.info("Running 'list' command.");

		ListOptions options;
		try {
			options = CliFrontendParser.parseListCommand(args);
		}
		catch (CliArgsException e) {
			return handleArgException(e);
		}
		catch (Throwable t) {
			return handleError(t);
		}

		// evaluate help flag
		if (options.isPrintHelp()) {
			CliFrontendParser.printHelpForList();
			return 0;
		}

		boolean running = options.getRunning();
		boolean scheduled = options.getScheduled();

		// print running and scheduled jobs if not option supplied
		if (!running && !scheduled) {
			running = true;
			scheduled = true;
		}

		try {
			ActorGateway jobManagerGateway = getJobManagerGateway(options);

			LOG.info("Connecting to JobManager to retrieve list of jobs");
			Future<Object> response = jobManagerGateway.ask(
				JobManagerMessages.getRequestRunningJobsStatus(),
				clientTimeout);

			Object result;
			try {
				result = Await.result(response, clientTimeout);
			}
			catch (Exception e) {
				throw new Exception("Could not retrieve running jobs from the JobManager.", e);
			}

			if (result instanceof RunningJobsStatus) {
				LOG.info("Successfully retrieved list of jobs");

				List<JobStatusMessage> jobs = ((RunningJobsStatus) result).getStatusMessages();

				ArrayList<JobStatusMessage> runningJobs = null;
				ArrayList<JobStatusMessage> scheduledJobs = null;
				if (running) {
					runningJobs = new ArrayList<JobStatusMessage>();
				}
				if (scheduled) {
					scheduledJobs = new ArrayList<JobStatusMessage>();
				}

				for (JobStatusMessage rj : jobs) {
					if (running && (rj.getJobState().equals(JobStatus.RUNNING)
							|| rj.getJobState().equals(JobStatus.RESTARTING))) {
						runningJobs.add(rj);
					}
					if (scheduled && rj.getJobState().equals(JobStatus.CREATED)) {
						scheduledJobs.add(rj);
					}
				}

				SimpleDateFormat df = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss");
				Comparator<JobStatusMessage> njec = new Comparator<JobStatusMessage>(){
					@Override
					public int compare(JobStatusMessage o1, JobStatusMessage o2) {
						return (int) (o1.getStartTime() - o2.getStartTime());
					}
				};

				if (running) {
					if (runningJobs.size() == 0) {
						System.out.println("No running jobs.");
					}
					else {
						Collections.sort(runningJobs, njec);

						System.out.println("------------------ Running/Restarting Jobs -------------------");
						for (JobStatusMessage rj : runningJobs) {
							System.out.println(df.format(new Date(rj.getStartTime()))
									+ " : " + rj.getJobId() + " : " + rj.getJobName() + " (" + rj.getJobState() + ")");
						}
						System.out.println("--------------------------------------------------------------");
					}
				}
				if (scheduled) {
					if (scheduledJobs.size() == 0) {
						System.out.println("No scheduled jobs.");
					}
					else {
						Collections.sort(scheduledJobs, njec);

						System.out.println("----------------------- Scheduled Jobs -----------------------");
						for (JobStatusMessage rj : scheduledJobs) {
							System.out.println(df.format(new Date(rj.getStartTime()))
									+ " : " + rj.getJobId() + " : " + rj.getJobName());
						}
						System.out.println("--------------------------------------------------------------");
					}
				}
				return 0;
			}
			else {
				throw new Exception("ReqeustRunningJobs requires a response of type " +
						"RunningJobs. Instead the response is of type " + result.getClass() + ".");
			}
		}
		catch (Throwable t) {
			return handleError(t);
		}
	}

	/**
	 * Executes the STOP action.
	 *
	 * @param args Command line arguments for the stop action.
	 */
	protected int stop(String[] args) {
		LOG.info("Running 'stop' command.");

		StopOptions options;
		try {
			options = CliFrontendParser.parseStopCommand(args);
		}
		catch (CliArgsException e) {
			return handleArgException(e);
		}
		catch (Throwable t) {
			return handleError(t);
		}

		// evaluate help flag
		if (options.isPrintHelp()) {
			CliFrontendParser.printHelpForStop();
			return 0;
		}

		String[] stopArgs = options.getArgs();
		JobID jobId;

		if (stopArgs.length > 0) {
			String jobIdString = stopArgs[0];
			try {
				jobId = new JobID(StringUtils.hexStringToByte(jobIdString));
			}
			catch (Exception e) {
				return handleError(e);
			}
		}
		else {
			return handleArgException(new CliArgsException("Missing JobID"));
		}

		try {
			ActorGateway jobManager = getJobManagerGateway(options);
			Future<Object> response = jobManager.ask(new StopJob(jobId), clientTimeout);

			final Object rc = Await.result(response, clientTimeout);

			if (rc instanceof StoppingFailure) {
				throw new Exception("Stopping the job with ID " + jobId + " failed.",
						((StoppingFailure) rc).cause());
			}

			return 0;
		}
		catch (Throwable t) {
			return handleError(t);
		}
	}

	/**
	 * Executes the CANCEL action.
	 *
	 * @param args Command line arguments for the cancel action.
	 */
	protected int cancel(String[] args) {
		LOG.info("Running 'cancel' command.");

		CancelOptions options;
		try {
			options = CliFrontendParser.parseCancelCommand(args);
		}
		catch (CliArgsException e) {
			return handleArgException(e);
		}
		catch (Throwable t) {
			return handleError(t);
		}

		// evaluate help flag
		if (options.isPrintHelp()) {
			CliFrontendParser.printHelpForCancel();
			return 0;
		}

		String[] cleanedArgs = options.getArgs();

		boolean withSavepoint = options.isWithSavepoint();
		String targetDirectory = options.getSavepointTargetDirectory();

		JobID jobId;

		// Figure out jobID. This is a little overly complicated, because
		// we have to figure out whether the optional target directory
		// is set:
		// - cancel -s <jobID> => default target dir (JobID parsed as opt arg)
		// - cancel -s <targetDir> <jobID> => custom target dir (parsed correctly)
		if (cleanedArgs.length > 0) {
			String jobIdString = cleanedArgs[0];
			try {
				jobId = new JobID(StringUtils.hexStringToByte(jobIdString));
			} catch (Exception e) {
				LOG.error("Error: The value for the Job ID is not a valid ID.");
				System.out.println("Error: The value for the Job ID is not a valid ID.");
				return 1;
			}
		} else if (targetDirectory != null)  {
			// Try this for case: cancel -s <jobID> (default savepoint target dir)
			String jobIdString = targetDirectory;
			try {
				jobId = new JobID(StringUtils.hexStringToByte(jobIdString));
				targetDirectory = null;
			} catch (Exception e) {
				LOG.error("Missing JobID in the command line arguments.");
				System.out.println("Error: Specify a Job ID to cancel a job.");
				return 1;
			}
		} else {
			LOG.error("Missing JobID in the command line arguments.");
			System.out.println("Error: Specify a Job ID to cancel a job.");
			return 1;
		}

		try {
			ActorGateway jobManager = getJobManagerGateway(options);

			Object cancelMsg;
			if (withSavepoint) {
				if (targetDirectory == null) {
					logAndSysout("Cancelling job " + jobId + " with savepoint to default savepoint directory.");
				} else {
					logAndSysout("Cancelling job " + jobId + " with savepoint to " + targetDirectory + ".");
				}
				cancelMsg = new CancelJobWithSavepoint(jobId, targetDirectory);
			} else {
				logAndSysout("Cancelling job " + jobId + ".");
				cancelMsg = new CancelJob(jobId);
			}

			Future<Object> response = jobManager.ask(cancelMsg, clientTimeout);
			final Object rc = Await.result(response, clientTimeout);

			if (rc instanceof CancellationSuccess) {
				if (withSavepoint) {
					CancellationSuccess success = (CancellationSuccess) rc;
					String savepointPath = success.savepointPath();
					logAndSysout("Cancelled job " + jobId + ". Savepoint stored in " + savepointPath + ".");
				} else {
					logAndSysout("Cancelled job " + jobId + ".");
				}
			} else if (rc instanceof CancellationFailure) {
				throw new Exception("Canceling the job with ID " + jobId + " failed.",
						((CancellationFailure) rc).cause());
			} else {
				throw new IllegalStateException("Unexpected response: " + rc);
			}

			return 0;
		}
		catch (Throwable t) {
			return handleError(t);
		}
	}

	/**
	 * Executes the SAVEPOINT action.
	 *
	 * @param args Command line arguments for the cancel action.
	 */
	protected int savepoint(String[] args) {
		LOG.info("Running 'savepoint' command.");

		SavepointOptions options;
		try {
			options = CliFrontendParser.parseSavepointCommand(args);
		} catch (CliArgsException e) {
			return handleArgException(e);
		} catch (Throwable t) {
			return handleError(t);
		}

		// evaluate help flag
		if (options.isPrintHelp()) {
			CliFrontendParser.printHelpForSavepoint();
			return 0;
		}

		if (options.isDispose()) {
			// Discard
			return disposeSavepoint(options);
		} else {
			// Trigger
			String[] cleanedArgs = options.getArgs();
			JobID jobId;

			if (cleanedArgs.length >= 1) {
				String jobIdString = cleanedArgs[0];
				try {
					jobId = new JobID(StringUtils.hexStringToByte(jobIdString));
				} catch (Exception e) {
					return handleArgException(new IllegalArgumentException(
							"Error: The value for the Job ID is not a valid ID."));
				}
			} else {
				return handleArgException(new IllegalArgumentException(
						"Error: The value for the Job ID is not a valid ID. " +
								"Specify a Job ID to trigger a savepoint."));
			}

			String savepointDirectory = null;
			if (cleanedArgs.length >= 2) {
				savepointDirectory = cleanedArgs[1];
			}

			// Print superfluous arguments
			if (cleanedArgs.length >= 3) {
				logAndSysout("Provided more arguments than required. Ignoring not needed arguments.");
			}

			return triggerSavepoint(options, jobId, savepointDirectory);
		}
	}

	/**
	 * Sends a {@link org.apache.flink.runtime.messages.JobManagerMessages.TriggerSavepoint}
	 * message to the job manager.
	 */
	private int triggerSavepoint(SavepointOptions options, JobID jobId, String savepointDirectory) {
		try {
			ActorGateway jobManager = getJobManagerGateway(options);

			logAndSysout("Triggering savepoint for job " + jobId + ".");
			Future<Object> response = jobManager.ask(new TriggerSavepoint(jobId, Option.apply(savepointDirectory)),
					new FiniteDuration(1, TimeUnit.HOURS));

			Object result;
			try {
				logAndSysout("Waiting for response...");
				result = Await.result(response, FiniteDuration.Inf());
			}
			catch (Exception e) {
				throw new Exception("Triggering a savepoint for the job " + jobId + " failed.", e);
			}

			if (result instanceof TriggerSavepointSuccess) {
				TriggerSavepointSuccess success = (TriggerSavepointSuccess) result;
				logAndSysout("Savepoint completed. Path: " + success.savepointPath());
				logAndSysout("You can resume your program from this savepoint with the run command.");

				return 0;
			}
			else if (result instanceof TriggerSavepointFailure) {
				TriggerSavepointFailure failure = (TriggerSavepointFailure) result;
				throw failure.cause();
			}
			else {
				throw new IllegalStateException("Unknown JobManager response of type " +
						result.getClass());
			}
		}
		catch (Throwable t) {
			return handleError(t);
		}
	}

	/**
	 * Sends a {@link org.apache.flink.runtime.messages.JobManagerMessages.DisposeSavepoint}
	 * message to the job manager.
	 */
	private int disposeSavepoint(SavepointOptions options) {
		try {
			String savepointPath = options.getSavepointPath();
			if (savepointPath == null) {
				throw new IllegalArgumentException("Missing required argument: savepoint path. " +
						"Usage: bin/flink savepoint -d <savepoint-path>");
			}

			ActorGateway jobManager = getJobManagerGateway(options);

			logAndSysout("Disposing savepoint '" + savepointPath + "'.");

			Object msg = new DisposeSavepoint(savepointPath);
			Future<Object> response = jobManager.ask(msg, clientTimeout);

			Object result;
			try {
				logAndSysout("Waiting for response...");
				result = Await.result(response, clientTimeout);
			} catch (Exception e) {
				throw new Exception("Disposing the savepoint with path" + savepointPath + " failed.", e);
			}

			if (result.getClass() == JobManagerMessages.getDisposeSavepointSuccess().getClass()) {
				logAndSysout("Savepoint '" + savepointPath + "' disposed.");
				return 0;
			} else if (result instanceof DisposeSavepointFailure) {
				DisposeSavepointFailure failure = (DisposeSavepointFailure) result;

				if (failure.cause() instanceof ClassNotFoundException) {
					throw new ClassNotFoundException("Savepoint disposal failed, because of a " +
							"missing class. This is most likely caused by a custom state " +
							"instance, which cannot be disposed without the user code class " +
							"loader. Please provide the program jar with which you have created " +
							"the savepoint via -j <JAR> for disposal.",
							failure.cause().getCause());
				} else {
					throw failure.cause();
				}
			} else {
				throw new IllegalStateException("Unknown JobManager response of type " +
						result.getClass());
			}
		} catch (Throwable t) {
			return handleError(t);
		}
	}

	// --------------------------------------------------------------------------------------------
	//  Interaction with programs and JobManager
	// --------------------------------------------------------------------------------------------

	protected int executeProgram(PackagedProgram program, ClusterClient client, int parallelism) {
		logAndSysout("Starting execution of program");

		JobSubmissionResult result;
		try {
			result = client.run(program, parallelism);
		} catch (ProgramParametrizationException e) {
			return handleParametrizationException(e);
		} catch (ProgramMissingJobException e) {
			return handleMissingJobException();
		} catch (ProgramInvocationException e) {
			return handleError(e);
		} finally {
			program.deleteExtractedLibraries();
		}

		if (null == result) {
			logAndSysout("No JobSubmissionResult returned, please make sure you called " +
				"ExecutionEnvironment.execute()");
			return 1;
		}

		if (result.isJobExecutionResult()) {
			logAndSysout("Program execution finished");
			JobExecutionResult execResult = result.getJobExecutionResult();
			System.out.println("Job with JobID " + execResult.getJobID() + " has finished.");
			System.out.println("Job Runtime: " + execResult.getNetRuntime() + " ms");
			Map<String, Object> accumulatorsResult = execResult.getAllAccumulatorResults();
			if (accumulatorsResult.size() > 0) {
				System.out.println("Accumulator Results: ");
				System.out.println(AccumulatorHelper.getResultsFormatted(accumulatorsResult));
			}
		} else {
			logAndSysout("Job has been submitted with JobID " + result.getJobID());
		}

		return 0;
	}

	/**
	 * Creates a Packaged program from the given command line options.
	 *
	 * @return A PackagedProgram (upon success)
	 * @throws java.io.FileNotFoundException
	 * @throws org.apache.flink.client.program.ProgramInvocationException
	 */
	protected PackagedProgram buildProgram(ProgramOptions options)
			throws FileNotFoundException, ProgramInvocationException {
		String[] programArgs = options.getProgramArgs();
		String jarFilePath = options.getJarFilePath();
		List<URL> classpaths = options.getClasspaths();

		if (jarFilePath == null) {
			throw new IllegalArgumentException("The program JAR file was not specified.");
		}

		File jarFile = new File(jarFilePath);

		// Check if JAR file exists
		if (!jarFile.exists()) {
			throw new FileNotFoundException("JAR file does not exist: " + jarFile);
		}
		else if (!jarFile.isFile()) {
			throw new FileNotFoundException("JAR file is not a file: " + jarFile);
		}

		// Get assembler class
		String entryPointClass = options.getEntryPointClassName();

		PackagedProgram program = entryPointClass == null ?
				new PackagedProgram(jarFile, classpaths, programArgs) :
				new PackagedProgram(jarFile, classpaths, entryPointClass, programArgs);

		program.setSavepointRestoreSettings(options.getSavepointRestoreSettings());

		return program;
	}

	/**
	 * Updates the associated configuration with the given command line options.
	 *
	 * @param options Command line options
	 */
	protected ClusterClient retrieveClient(CommandLineOptions options) {
		CustomCommandLine customCLI = getActiveCustomCommandLine(options.getCommandLine());
		try {
			ClusterClient client = customCLI.retrieveCluster(options.getCommandLine(), config);
			logAndSysout("Using address " + client.getJobManagerAddress() + " to connect to JobManager.");
			return client;
		} catch (Exception e) {
			LOG.error("Couldn't retrieve {} cluster.", customCLI.getId(), e);
			throw new IllegalConfigurationException("Couldn't retrieve client for cluster", e);
		}
	}

	/**
	 * Retrieves the {@link ActorGateway} for the JobManager. The ClusterClient is retrieved
	 * from the provided {@link CommandLineOptions}.
	 *
	 * @param options CommandLineOptions specifying the JobManager URL
	 * @return Gateway to the JobManager
	 * @throws Exception
	 */
	protected ActorGateway getJobManagerGateway(CommandLineOptions options) throws Exception {
		logAndSysout("Retrieving JobManager.");
		return retrieveClient(options).getJobManagerGateway();
	}

	/**
	 * Creates a {@link ClusterClient} object from the given command line options and other parameters.
	 * @param options Command line options
	 * @param program The program for which to create the client.
	 * @throws Exception
	 */
	protected ClusterClient createClient(
			CommandLineOptions options,
			PackagedProgram program) throws Exception {

		// Get the custom command-line (e.g. Standalone/Yarn/Mesos)
		CustomCommandLine<?> activeCommandLine = getActiveCustomCommandLine(options.getCommandLine());

		ClusterClient client;
		try {
			client = activeCommandLine.retrieveCluster(options.getCommandLine(), config);
			logAndSysout("Cluster configuration: " + client.getClusterIdentifier());
		} catch (UnsupportedOperationException e) {
			try {
				String applicationName = "Flink Application: " + program.getMainClassName();
				client = activeCommandLine.createCluster(
					applicationName,
					options.getCommandLine(),
					config,
					program.getAllLibraries());
				logAndSysout("Cluster started: " + client.getClusterIdentifier());
			} catch (UnsupportedOperationException e2) {
				throw new IllegalConfigurationException(
					"The JobManager address is neither provided at the command-line, " +
						"nor configured in flink-conf.yaml.");
			}
		}

		// Avoid resolving the JobManager Gateway here to prevent blocking until we invoke the user's program.
		final InetSocketAddress jobManagerAddress = client.getJobManagerAddress();
		logAndSysout("Using address " + jobManagerAddress.getHostString() + ":" + jobManagerAddress.getPort() + " to connect to JobManager.");
		logAndSysout("JobManager web interface address " + client.getWebInterfaceURL());
		return client;
	}

	// --------------------------------------------------------------------------------------------
	//  Logging and Exception Handling
	// --------------------------------------------------------------------------------------------

	/**
	 * Displays an exception message for incorrect command line arguments.
	 *
	 * @param e The exception to display.
	 * @return The return code for the process.
	 */
	private int handleArgException(Exception e) {
		LOG.error("Invalid command line arguments. " + (e.getMessage() == null ? "" : e.getMessage()));

		System.out.println(e.getMessage());
		System.out.println();
		System.out.println("Use the help option (-h or --help) to get help on the command.");
		return 1;
	}

	/**
	 * Displays an optional exception message for incorrect program parametrization.
	 *
	 * @param e The exception to display.
	 * @return The return code for the process.
	 */
	private int handleParametrizationException(ProgramParametrizationException e) {
		System.err.println(e.getMessage());
		return 1;
	}

	/**
	 * Displays a message for a program without a job to execute.
	 *
	 * @return The return code for the process.
	 */
	private int handleMissingJobException() {
		System.err.println();
		System.err.println("The program didn't contain a Flink job. " +
			"Perhaps you forgot to call execute() on the execution environment.");
		return 1;
	}

	/**
	 * Displays an exception message.
	 *
	 * @param t The exception to display.
	 * @return The return code for the process.
	 */
	private int handleError(Throwable t) {
		LOG.error("Error while running the command.", t);

		System.err.println();
		System.err.println("------------------------------------------------------------");
		System.err.println(" The program finished with the following exception:");
		System.err.println();

		if (t.getCause() instanceof InvalidProgramException) {
			System.err.println(t.getCause().getMessage());
			StackTraceElement[] trace = t.getCause().getStackTrace();
			for (StackTraceElement ele: trace) {
				System.err.println("\t" + ele.toString());
				if (ele.getMethodName().equals("main")) {
					break;
				}
			}
		} else {
			t.printStackTrace();
		}
		return 1;
	}

	private void logAndSysout(String message) {
		LOG.info(message);
		System.out.println(message);
	}

	// --------------------------------------------------------------------------------------------
	//  Entry point for executable
	// --------------------------------------------------------------------------------------------

	/**
	 * Parses the command line arguments and starts the requested action.
	 *
	 * @param args command line arguments of the client.
	 * @return The return code of the program
	 */
	public int parseParameters(String[] args) {

		// check for action
		if (args.length < 1) {
			CliFrontendParser.printHelp();
			System.out.println("Please specify an action.");
			return 1;
		}

		// get action
		String action = args[0];

		// remove action from parameters
		final String[] params = Arrays.copyOfRange(args, 1, args.length);

		// do action
		switch (action) {
			case ACTION_RUN:
				return run(params);
			case ACTION_LIST:
				return list(params);
			case ACTION_INFO:
				return info(params);
			case ACTION_CANCEL:
				return cancel(params);
			case ACTION_STOP:
				return stop(params);
			case ACTION_SAVEPOINT:
				return savepoint(params);
			case "-h":
			case "--help":
				CliFrontendParser.printHelp();
				return 0;
			case "-v":
			case "--version":
				String version = EnvironmentInformation.getVersion();
				String commitID = EnvironmentInformation.getRevisionInformation().commitId;
				System.out.print("Version: " + version);
				System.out.println(!commitID.equals(EnvironmentInformation.UNKNOWN) ? ", Commit ID: " + commitID : "");
				return 0;
			default:
				System.out.printf("\"%s\" is not a valid action.\n", action);
				System.out.println();
				System.out.println("Valid actions are \"run\", \"list\", \"info\", \"savepoint\", \"stop\", or \"cancel\".");
				System.out.println();
				System.out.println("Specify the version option (-v or --version) to print Flink version.");
				System.out.println();
				System.out.println("Specify the help option (-h or --help) to get help on the command.");
				return 1;
		}
	}

	/**
	 * Submits the job based on the arguments.
	 */
	public static void main(final String[] args) {
		EnvironmentInformation.logEnvironmentInfo(LOG, "Command Line Client", args);

		try {
			final CliFrontend cli = new CliFrontend();
			SecurityUtils.install(new SecurityUtils.SecurityConfiguration(cli.config));
			int retCode = SecurityUtils.getInstalledContext()
					.runSecured(new Callable<Integer>() {
						@Override
						public Integer call() {
							return cli.parseParameters(args);
						}
					});
			System.exit(retCode);
		}
		catch (Throwable t) {
			LOG.error("Fatal error while running command line interface.", t);
			t.printStackTrace();
			System.exit(31);
		}
	}

	// --------------------------------------------------------------------------------------------
	//  Miscellaneous Utilities
	// --------------------------------------------------------------------------------------------

	public static String getConfigurationDirectoryFromEnv() {
		String location = System.getenv(ConfigConstants.ENV_FLINK_CONF_DIR);

		if (location != null) {
			if (new File(location).exists()) {
				return location;
			}
			else {
				throw new RuntimeException("The config directory '" + location + "', specified in the '" +
					ConfigConstants.ENV_FLINK_CONF_DIR + "' environment variable, does not exist.");
			}
		}
		else if (new File(CONFIG_DIRECTORY_FALLBACK_1).exists()) {
			location = CONFIG_DIRECTORY_FALLBACK_1;
		}
		else if (new File(CONFIG_DIRECTORY_FALLBACK_2).exists()) {
			location = CONFIG_DIRECTORY_FALLBACK_2;
		}
		else {
			throw new RuntimeException("The configuration directory was not specified. " +
					"Please specify the directory containing the configuration file through the '" +
				ConfigConstants.ENV_FLINK_CONF_DIR + "' environment variable.");
		}
		return location;
	}

	/**
	 * Writes the given job manager address to the associated configuration object.
	 *
	 * @param address Address to write to the configuration
	 * @param config The config to write to
	 */
	public static void setJobManagerAddressInConfig(Configuration config, InetSocketAddress address) {
		config.setString(JobManagerOptions.ADDRESS, address.getHostString());
		config.setInteger(JobManagerOptions.PORT, address.getPort());
	}

	// --------------------------------------------------------------------------------------------
	//  Custom command-line
	// --------------------------------------------------------------------------------------------

	/**
	 * Gets the custom command-line for the arguments.
	 * @param commandLine The input to the command-line.
	 * @return custom command-line which is active (may only be one at a time)
	 */
	public CustomCommandLine getActiveCustomCommandLine(CommandLine commandLine) {
		for (CustomCommandLine cli : customCommandLines) {
			if (cli.isActive(commandLine, config)) {
				return cli;
			}
		}
		throw new IllegalStateException("No command-line ran.");
	}

	/**
	 * Retrieves the loaded custom command-lines.
	 * @return An unmodifiyable list of loaded custom command-lines.
	 */
	public static List<CustomCommandLine<?>> getCustomCommandLineList() {
		return Collections.unmodifiableList(customCommandLines);
	}

	/**
	 * Loads a class from the classpath that implements the CustomCommandLine interface.
	 * @param className The fully-qualified class name to load.
	 * @param params The constructor parameters
	 */
	private static CustomCommandLine<?> loadCustomCommandLine(String className, Object... params) throws IllegalAccessException, InvocationTargetException, InstantiationException, ClassNotFoundException, NoSuchMethodException {

		Class<? extends CustomCommandLine> customCliClass =
			Class.forName(className).asSubclass(CustomCommandLine.class);

		// construct class types from the parameters
		Class<?>[] types = new Class<?>[params.length];
		for (int i = 0; i < params.length; i++) {
			Preconditions.checkNotNull(params[i], "Parameters for custom command-lines may not be null.");
			types[i] = params[i].getClass();
		}

		Constructor<? extends CustomCommandLine> constructor = customCliClass.getConstructor(types);

		return constructor.newInstance(params);
	}

}


text:
from django.db import models, migrations

def remove_manage_scripts(apps, schema_editor):
    ScriptDB = apps.get_model("scripts", "ScriptDB")
    for script in ScriptDB.objects.filter(
        db_typeclass_path__in=(
            "evennia.scripts.scripts.CheckSessions",
            "evennia.scripts.scripts.ValidateScripts",
            "evennia.scripts.scripts.ValidateChannelHandler",
            "evennia.scripts.scripts.ValidateIdmapperCache",
            "evennia.utils.gametime.GameTime",
        )
    ):
        script.delete()

class Migration(migrations.Migration):

    dependencies = [
        (
            "scripts",
            "0003_checksessions_defaultscript_donothing_scriptbase_store_validatechannelhandler_validateidmappercache_",
        )
    ]

    operations = [migrations.RunPython(remove_manage_scripts)]

redpajama_set_name: RedPajamaGithub


text:
package util

import (
	"context"
	"encoding/json"
	"fmt"
	"time"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/apimachinery/pkg/util/strategicpatch"
	"k8s.io/client-go/kubernetes"
	corev1helpers "k8s.io/component-helpers/scheduling/corev1"
	"k8s.io/klog/v2"
	extenderv1 "k8s.io/kube-scheduler/extender/v1"
	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
)

// GetPodFullName returns a name that uniquely identifies a pod.
func GetPodFullName(pod *v1.Pod) string {
	// Use underscore as the delimiter because it is not allowed in pod name
	// (DNS subdomain format).
	return pod.Name + "_" + pod.Namespace
}

// GetPodStartTime returns start time of the given pod or current timestamp
// if it hasn't started yet.
func GetPodStartTime(pod *v1.Pod) *metav1.Time {
	if pod.Status.StartTime != nil {
		return pod.Status.StartTime
	}
	// Assumed pods and bound pods that haven't started don't have a StartTime yet.
	return &metav1.Time{Time: time.Now()}
}

// GetEarliestPodStartTime returns the earliest start time of all pods that
// have the highest priority among all victims.
func GetEarliestPodStartTime(victims *extenderv1.Victims) *metav1.Time {
	if len(victims.Pods) == 0 {
		// should not reach here.
		klog.ErrorS(fmt.Errorf("victims.Pods is empty. Should not reach here"), "")
		return nil
	}

	earliestPodStartTime := GetPodStartTime(victims.Pods[0])
	maxPriority := corev1helpers.PodPriority(victims.Pods[0])

	for _, pod := range victims.Pods {
		if corev1helpers.PodPriority(pod) == maxPriority {
			if GetPodStartTime(pod).Before(earliestPodStartTime) {
				earliestPodStartTime = GetPodStartTime(pod)
			}
		} else if corev1helpers.PodPriority(pod) > maxPriority {
			maxPriority = corev1helpers.PodPriority(pod)
			earliestPodStartTime = GetPodStartTime(pod)
		}
	}

	return earliestPodStartTime
}

// MoreImportantPod return true when priority of the first pod is higher than
// the second one. If two pods' priorities are equal, compare their StartTime.
// It takes arguments of the type "interface{}" to be used with SortableList,
// but expects those arguments to be *v1.Pod.
func MoreImportantPod(pod1, pod2 *v1.Pod) bool {
	p1 := corev1helpers.PodPriority(pod1)
	p2 := corev1helpers.PodPriority(pod2)
	if p1 != p2 {
		return p1 > p2
	}
	return GetPodStartTime(pod1).Before(GetPodStartTime(pod2))
}

// PatchPodStatus calculates the delta bytes change from <old.Status> to <newStatus>,
// and then submit a request to API server to patch the pod changes.
func PatchPodStatus(cs kubernetes.Interface, old *v1.Pod, newStatus *v1.PodStatus) error {
	if newStatus == nil {
		return nil
	}

	oldData, err := json.Marshal(v1.Pod{Status: old.Status})
	if err != nil {
		return err
	}

	newData, err := json.Marshal(v1.Pod{Status: *newStatus})
	if err != nil {
		return err
	}
	patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, &v1.Pod{})
	if err != nil {
		return fmt.Errorf("failed to create merge patch for pod %q/%q: %v", old.Namespace, old.Name, err)
	}

	if "{}" == string(patchBytes) {
		return nil
	}

	_, err = cs.CoreV1().Pods(old.Namespace).Patch(context.TODO(), old.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status")
	return err
}

// DeletePod deletes the given <pod> from API server
func DeletePod(cs kubernetes.Interface, pod *v1.Pod) error {
	return cs.CoreV1().Pods(pod.Namespace).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{})
}

// ClearNominatedNodeName internally submit a patch request to API server
// to set each pods[*].Status.NominatedNodeName> to "".
func ClearNominatedNodeName(cs kubernetes.Interface, pods ...*v1.Pod) utilerrors.Aggregate {
	var errs []error
	for _, p := range pods {
		if len(p.Status.NominatedNodeName) == 0 {
			continue
		}
		podStatusCopy := p.Status.DeepCopy()
		podStatusCopy.NominatedNodeName = ""
		if err := PatchPodStatus(cs, p, podStatusCopy); err != nil {
			errs = append(errs, err)
		}
	}
	return utilerrors.NewAggregate(errs)
}

// IsScalarResourceName validates the resource for Extended, Hugepages, Native and AttachableVolume resources
func IsScalarResourceName(name v1.ResourceName) bool {
	return v1helper.IsExtendedResourceName(name) || v1helper.IsHugePageResourceName(name) ||
		v1helper.IsPrefixedNativeResource(name) || v1helper.IsAttachableVolumeResourceName(name)
}


text:
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "content/browser/prerender/prerender_host.h"

#include "base/feature_list.h"
#include "base/observer_list.h"
#include "base/run_loop.h"
#include "base/trace_event/common/trace_event_common.h"
#include "base/trace_event/trace_conversion_helper.h"
#include "base/trace_event/typed_macros.h"
#include "content/browser/prerender/prerender_host_registry.h"
#include "content/browser/prerender/prerender_metrics.h"
#include "content/browser/renderer_host/frame_tree.h"
#include "content/browser/renderer_host/frame_tree_node.h"
#include "content/browser/renderer_host/navigation_controller_impl.h"
#include "content/browser/renderer_host/navigation_entry_restore_context_impl.h"
#include "content/browser/renderer_host/navigation_request.h"
#include "content/browser/renderer_host/page_impl.h"
#include "content/browser/renderer_host/render_frame_host_impl.h"
#include "content/browser/renderer_host/render_frame_proxy_host.h"
#include "content/browser/renderer_host/render_view_host_impl.h"
#include "content/browser/site_info.h"
#include "content/browser/site_instance_impl.h"
#include "content/browser/web_contents/web_contents_impl.h"
#include "content/public/browser/back_forward_cache.h"
#include "content/public/browser/navigation_controller.h"
#include "content/public/browser/web_contents.h"
#include "content/public/browser/web_contents_delegate.h"
#include "content/public/browser/web_contents_observer.h"
#include "content/public/common/referrer.h"
#include "net/base/load_flags.h"
#include "third_party/blink/public/common/features.h"
#include "third_party/blink/public/mojom/loader/referrer.mojom.h"

namespace content {

namespace {

bool AreHttpRequestHeadersCompatible(
    const std::string& potential_activation_headers_str,
    const std::string& prerender_headers_str) {
  net::HttpRequestHeaders prerender_headers;
  prerender_headers.AddHeadersFromString(prerender_headers_str);

  net::HttpRequestHeaders potential_activation_headers;
  potential_activation_headers.AddHeadersFromString(
      potential_activation_headers_str);

  // `prerender_headers` contains the "Purpose: prefetch" to notify servers of
  // prerender requests, while `potential_activation_headers` doesn't contain
  // it. Remove "Purpose" matching from consideration so that activation works
  // with the header.
  prerender_headers.RemoveHeader("Purpose");
  potential_activation_headers.RemoveHeader("Purpose");

  return prerender_headers.ToString() ==
         potential_activation_headers.ToString();
}

}  // namespace

class PrerenderHost::PageHolder : public FrameTree::Delegate,
                                  public NavigationControllerDelegate {
 public:
  explicit PageHolder(WebContentsImpl& web_contents)
      : web_contents_(web_contents),
        frame_tree_(
            std::make_unique<FrameTree>(web_contents.GetBrowserContext(),
                                        this,
                                        this,
                                        &web_contents,
                                        &web_contents,
                                        &web_contents,
                                        &web_contents,
                                        &web_contents,
                                        &web_contents,
                                        FrameTree::Type::kPrerender)) {
    scoped_refptr<SiteInstance> site_instance =
        SiteInstance::Create(web_contents.GetBrowserContext());
    frame_tree_->Init(site_instance.get(),
                      /*renderer_initiated_creation=*/false,
                      /*main_frame_name=*/"", /*opener=*/nullptr,
                      /*frame_policy=*/blink::FramePolicy());

    // Use the same SessionStorageNamespace as the primary page for the
    // prerendering page.
    frame_tree_->controller().SetSessionStorageNamespace(
        site_instance->GetStoragePartitionConfig(),
        web_contents_.GetPrimaryFrameTree()
            .controller()
            .GetSessionStorageNamespace(
                site_instance->GetStoragePartitionConfig()));

    // TODO(https://crbug.com/1199679): This should be moved to FrameTree::Init
    web_contents_.NotifySwappedFromRenderManager(
        /*old_frame=*/nullptr,
        frame_tree_->root()->render_manager()->current_frame_host());
  }

  ~PageHolder() override {
    // If we are still waiting on test loop, we can assume the page loading step
    // has been cancelled and the PageHolder is being discarded without
    // completing loading the page.
    if (on_wait_loading_finished_)
      std::move(on_wait_loading_finished_)
          .Run(PrerenderHost::LoadingOutcome::kPrerenderingCancelled);

    if (frame_tree_)
      frame_tree_->Shutdown();
  }

  // FrameTree::Delegate

  // TODO(https://crbug.com/1199682): Correctly handle load events. Ignored for
  // now as it confuses WebContentsObserver instances because they can not
  // distinguish between the different FrameTrees.

  void DidStartLoading(FrameTreeNode* frame_tree_node,
                       bool should_show_loading_ui) override {}

  void DidStopLoading() override {
    if (on_wait_loading_finished_) {
      std::move(on_wait_loading_finished_)
          .Run(PrerenderHost::LoadingOutcome::kLoadingCompleted);
    }
  }

  void DidChangeLoadProgress() override {}
  bool IsHidden() override { return true; }
  FrameTree* LoadingTree() override {
    // For prerendering loading tree is the same as its frame tree as loading is
    // done at a frame tree level in the background, unlike the loading visible
    // to the user where we account for nested frame tree loading state.
    return frame_tree_.get();
  }
  void NotifyPageChanged(PageImpl& page) override {}
  int GetOuterDelegateFrameTreeNodeId() override {
    // A prerendered FrameTree is not "inner to" or "nested inside" another
    // FrameTree; it exists in parallel to the primary FrameTree of the current
    // WebContents. Therefore, it must not attempt to access the primary
    // FrameTree in the sense of an "outer delegate" relationship, so we return
    // the invalid ID here.
    return FrameTreeNode::kFrameTreeNodeInvalidId;
  }
  bool IsPortal() override { return false; }

  // NavigationControllerDelegate
  void NotifyNavigationStateChanged(InvalidateTypes changed_flags) override {}
  void NotifyBeforeFormRepostWarningShow() override {}
  void NotifyNavigationEntryCommitted(
      const LoadCommittedDetails& load_details) override {}
  void NotifyNavigationEntryChanged(
      const EntryChangedDetails& change_details) override {}
  void NotifyNavigationListPruned(
      const PrunedDetails& pruned_details) override {}
  void NotifyNavigationEntriesDeleted() override {}
  void ActivateAndShowRepostFormWarningDialog() override {
    // Not supported, cancel pending reload.
    GetNavigationController().CancelPendingReload();
  }
  bool ShouldPreserveAbortedURLs() override { return false; }
  WebContents* DeprecatedGetWebContents() override { return GetWebContents(); }
  void UpdateOverridingUserAgent() override {}

  NavigationControllerImpl& GetNavigationController() {
    return frame_tree_->controller();
  }

  WebContents* GetWebContents() { return &web_contents_; }

  FrameTree& GetPrimaryFrameTree() {
    return web_contents_.GetPrimaryFrameTree();
  }

  std::unique_ptr<StoredPage> Activate(NavigationRequest& navigation_request) {
    // There should be no ongoing main-frame navigation during activation.
    // TODO(https://crbug.com/1190644): Make sure sub-frame navigations are
    // fine.
    DCHECK(!frame_tree_->root()->HasNavigation());

    // Before the root's current_frame_host is cleared, collect the subframes of
    // `frame_tree_` whose FrameTree will need to be updated.
    FrameTree::NodeRange node_range = frame_tree_->Nodes();
    std::vector<FrameTreeNode*> subframe_nodes(std::next(node_range.begin()),
                                               node_range.end());

    // Before the root's current_frame_host is cleared, collect the replication
    // state so that it can be used for post-activation validation.
    blink::mojom::FrameReplicationState prior_replication_state =
        frame_tree_->root()->current_replication_state();

    // NOTE: TakePrerenderedPage() clears the current_frame_host value of
    // frame_tree_->root(). Do not add any code between here and
    // frame_tree_.reset() that calls into observer functions to minimize the
    // duration of current_frame_host being null.
    //
    // TODO(https://crbug.com/1176148): Investigate how to combine taking the
    // prerendered page and frame_tree_ destruction.
    std::unique_ptr<StoredPage> page =
        frame_tree_->root()->render_manager()->TakePrerenderedPage();

    std::unique_ptr<NavigationEntryRestoreContextImpl> context =
        std::make_unique<NavigationEntryRestoreContextImpl>();
    std::unique_ptr<NavigationEntryImpl> nav_entry =
        GetNavigationController()
            .GetEntryWithUniqueID(page->render_frame_host->nav_entry_id())
            ->CloneWithoutSharing(context.get());

    navigation_request.SetPrerenderActivationNavigationState(
        std::move(nav_entry), prior_replication_state);

    FrameTree& target_frame_tree = GetPrimaryFrameTree();
    DCHECK_EQ(&target_frame_tree,
              navigation_request.frame_tree_node()->frame_tree());

    // We support activating the prerenderd page only to the topmost
    // RenderFrameHost.
    CHECK(!page->render_frame_host->GetParentOrOuterDocumentOrEmbedder());

    page->render_frame_host->SetFrameTreeNode(*(target_frame_tree.root()));
    // Copy frame name into the replication state of the primary main frame to
    // ensure that the replication state of the primary main frame after
    // activation matches the replication state stored in the renderer.
    // TODO(https://crbug.com/1237091): Copying frame name here is suboptimal
    // and ideally we'd do this at the same time when transferring the proxies
    // from the StoredPage into RenderFrameHostManager. However, this is a
    // temporary solution until we move this into BrowsingContextState,
    // along with RenderFrameProxyHost.
    page->render_frame_host->frame_tree_node()->set_frame_name_for_activation(
        prior_replication_state.unique_name, prior_replication_state.name);
    for (auto& it : page->proxy_hosts) {
      it.second->set_frame_tree_node(*(target_frame_tree.root()));
    }

    // Iterate over the root RenderFrameHost's subframes and update the
    // associated frame tree. Note that subframe proxies don't need their
    // FrameTrees independently updated, since their FrameTreeNodes don't
    // change, and FrameTree references in those FrameTreeNodes will be updated
    // through RenderFrameHosts.
    //
    // TODO(https://crbug.com/1199693): Need to investigate if and how
    // pending delete RenderFrameHost objects should be handled if prerendering
    // runs all of the unload handlers; they are not currently handled here.
    // This is because pending delete RenderFrameHosts can still receive and
    // process some messages while the RenderFrameHost FrameTree and
    // FrameTreeNode are stale.
    for (FrameTreeNode* subframe_node : subframe_nodes) {
      subframe_node->SetFrameTree(target_frame_tree);
    }

    page->render_frame_host->ForEachRenderFrameHostIncludingSpeculative(
        base::BindRepeating(
            [](const WebContentsImpl& web_contents, RenderFrameHostImpl* rfh) {
              // The visibility state of the prerendering page has not been
              // updated by
              // WebContentsImpl::UpdateVisibilityAndNotifyPageAndView(). So
              // updates the visibility state using the PageVisibilityState of
              // |web_contents|.
              rfh->render_view_host()->SetFrameTreeVisibility(
                  web_contents.GetPageVisibilityState());
            },
            std::cref(web_contents_)));

    frame_tree_->Shutdown();
    frame_tree_.reset();

    return page;
  }

  PrerenderHost::LoadingOutcome WaitForLoadCompletionForTesting() {
    PrerenderHost::LoadingOutcome status =
        PrerenderHost::LoadingOutcome::kLoadingCompleted;
    if (!frame_tree_->IsLoading())
      return status;

    base::RunLoop loop;
    on_wait_loading_finished_ =
        base::BindOnce(&PrerenderHost::PageHolder::FinishWaitingForTesting,
                       loop.QuitClosure(), &status);
    loop.Run();
    return status;
  }

  FrameTree* frame_tree() { return frame_tree_.get(); }

 private:
  static void FinishWaitingForTesting(base::OnceClosure on_close,  // IN-TEST
                                      PrerenderHost::LoadingOutcome* result,
                                      PrerenderHost::LoadingOutcome status) {
    *result = status;
    std::move(on_close).Run();
  }

  // WebContents where this prerenderer is embedded.
  WebContentsImpl& web_contents_;

  // Used for testing, this closure is only set when waiting a page to be
  // either loaded for pre-rendering. |frame_tree_| provides us with a trigger
  // for when the page is loaded.
  base::OnceCallback<void(PrerenderHost::LoadingOutcome)>
      on_wait_loading_finished_;

  // Frame tree created for the prerenderer to load the page and prepare it for
  // a future activation. During activation, the prerendered page will be taken
  // out from |frame_tree_| and moved over to |web_contents_|'s primary frame
  // tree, while |frame_tree_| will be deleted.
  std::unique_ptr<FrameTree> frame_tree_;
};

PrerenderHost::PrerenderHost(const PrerenderAttributes& attributes,
                             WebContents& web_contents)
    : attributes_(attributes) {
  DCHECK(blink::features::IsPrerender2Enabled());
  CreatePageHolder(*static_cast<WebContentsImpl*>(&web_contents));
}

PrerenderHost::~PrerenderHost() {
  // Stop observing here. Otherwise, destructing members may lead
  // DidFinishNavigation call after almost everything being destructed.
  Observe(nullptr);

  for (auto& observer : observers_)
    observer.OnHostDestroyed();

  if (!final_status_)
    RecordFinalStatus(FinalStatus::kDestroyed, attributes_.initiator_ukm_id,
                      ukm::kInvalidSourceId);
}

// TODO(https://crbug.com/1132746): Inspect diffs from the current
// no-state-prefetch implementation. See PrerenderContents::StartPrerendering()
// for example.
bool PrerenderHost::StartPrerendering() {
  TRACE_EVENT0("navigation", "PrerenderHost::StartPrerendering");

  // Observe events about the prerendering contents.
  Observe(page_holder_->GetWebContents());

  // Start prerendering navigation.
  NavigationController::LoadURLParams load_url_params(
      attributes_.prerendering_url);
  load_url_params.initiator_origin = attributes_.initiator_origin;
  load_url_params.initiator_process_id = attributes_.initiator_process_id;
  load_url_params.initiator_frame_token = attributes_.initiator_frame_token;
  load_url_params.is_renderer_initiated = !attributes_.IsBrowserInitiated();
  load_url_params.transition_type =
      ui::PageTransitionFromInt(attributes_.transition_type);

  // Just use the referrer from attributes, as NoStatePrefetch does.
  // TODO(crbug.com/1176054): For cross-origin prerender, follow the spec steps
  // for "sufficiently-strict speculative navigation referrer policies".
  load_url_params.referrer = attributes_.referrer;

  // TODO(https://crbug.com/1189034): Should we set `override_user_agent` here?
  // Things seem to work without it.

  // TODO(https://crbug.com/1132746): Set up other fields of `load_url_params`
  // as well, and add tests for them.
  base::WeakPtr<NavigationHandle> created_navigation_handle =
      page_holder_->GetNavigationController().LoadURLWithParams(
          load_url_params);

  if (!created_navigation_handle)
    return false;

  if (initial_navigation_id_.has_value()) {
    // In usual code path, `initial_navigation_id_` should be set by
    // PrerenderNavigationThrottle during `LoadURLWithParams` above.
    DCHECK_EQ(*initial_navigation_id_,
              created_navigation_handle->GetNavigationId());
    DCHECK(begin_params_);
    DCHECK(common_params_);
  } else {
    // In some exceptional code path, such as the navigation failed due to CSP
    // violations, PrerenderNavigationThrottle didn't run at this point. So,
    // set the ID here.
    initial_navigation_id_ = created_navigation_handle->GetNavigationId();
    // |begin_params_| and |common_params_| is null here, but it doesn't matter
    // as this branch is reached only when the initial navigation fails,
    // so this PrerenderHost can't be activated.
  }

  NavigationRequest* navigation_request =
      NavigationRequest::From(created_navigation_handle.get());
  // The initial navigation in the prerender frame tree should not wait for
  // `beforeunload` in the old page, so BeginNavigation stage should be reached
  // synchronously.
  DCHECK_GE(navigation_request->state(),
            NavigationRequest::WAITING_FOR_RENDERER_RESPONSE);
  return true;
}

void PrerenderHost::DidFinishNavigation(NavigationHandle* navigation_handle) {
  auto* navigation_request = NavigationRequest::From(navigation_handle);

  if (navigation_request->IsSameDocument())
    return;

  // Observe navigation only in the prerendering frame tree.
  if (navigation_request->frame_tree_node()->frame_tree() !=
      page_holder_->frame_tree()) {
    return;
  }

  const bool is_prerender_main_frame =
      navigation_request->GetFrameTreeNodeId() == frame_tree_node_id_;

  // Cancel prerendering on navigation request failure.
  //
  // Check net::Error here rather than PrerenderNavigationThrottle as CSP
  // blocking occurs before NavigationThrottles so cannot be observed in
  // NavigationThrottle::WillFailRequest().
  net::Error net_error = navigation_request->GetNetErrorCode();
  absl::optional<FinalStatus> status;
  if (net_error == net::Error::ERR_BLOCKED_BY_CSP) {
    status = FinalStatus::kNavigationRequestBlockedByCsp;
  } else if (net_error == net::Error::ERR_BLOCKED_BY_CLIENT) {
    status = FinalStatus::kBlockedByClient;
  } else if (is_prerender_main_frame && net_error != net::Error::OK) {
    status = FinalStatus::kNavigationRequestNetworkError;
  } else if (is_prerender_main_frame && !navigation_request->HasCommitted()) {
    status = FinalStatus::kNavigationNotCommitted;
  }
  if (status.has_value()) {
    Cancel(*status);
    return;
  }

  // The prerendered contents are considered ready for activation when the
  // main frame navigation reaches DidFinishNavigation.
  if (is_prerender_main_frame) {
    DCHECK(!is_ready_for_activation_);
    is_ready_for_activation_ = true;
  }
}

void PrerenderHost::OnVisibilityChanged(Visibility visibility) {
  TRACE_EVENT("navigation", "PrerenderHost::OnVisibilityChanged");
  if (visibility == Visibility::HIDDEN) {
    Cancel(FinalStatus::kTriggerBackgrounded);
  }
}

void PrerenderHost::ResourceLoadComplete(
    RenderFrameHost* render_frame_host,
    const GlobalRequestID& request_id,
    const blink::mojom::ResourceLoadInfo& resource_load_info) {
  // Observe resource loads only in the prerendering frame tree.
  if (&render_frame_host->GetPage() !=
      &GetPrerenderedMainFrameHost()->GetPage()) {
    return;
  }

  if (resource_load_info.net_error == net::Error::ERR_BLOCKED_BY_CLIENT) {
    Cancel(FinalStatus::kBlockedByClient);
  }
}

std::unique_ptr<StoredPage> PrerenderHost::Activate(
    NavigationRequest& navigation_request) {
  TRACE_EVENT1("navigation", "PrerenderHost::Activate", "navigation_request",
               &navigation_request);

  DCHECK(is_ready_for_activation_);
  is_ready_for_activation_ = false;

  std::unique_ptr<StoredPage> page = page_holder_->Activate(navigation_request);

  for (auto& observer : observers_)
    observer.OnActivated();

  // TODO(crbug.com/1299330): Replace
  // `navigation_request.GetNextPageUkmSourceId()` with prerendered page's UKM
  // source ID.
  RecordFinalStatus(FinalStatus::kActivated, attributes_.initiator_ukm_id,
                    navigation_request.GetNextPageUkmSourceId());
  return page;
}

// Ensure that the frame policies are compatible between primary main frame and
// prerendering main frame:
// a) primary main frame's pending_frame_policy would normally apply to the new
// document during its creation. However, for prerendering we can't apply it as
// the document is already created.
// b) prerender main frame's pending_frame_policy can't be transferred to the
// primary main frame, we should not activate if it's non-zero.
// c) Existing  document can't change the frame_policy it is affected by, so we
// can't transfer RenderFrameHosts between FrameTreeNodes with different frame
// policies.
//
// Usually frame policy for the main frame is empty as in the most common case a
// parent document sets a policy on the child iframe.
bool PrerenderHost::IsFramePolicyCompatibleWithPrimaryFrameTree() {
  FrameTreeNode* prerender_root_ftn = page_holder_->frame_tree()->root();
  FrameTreeNode* primary_root_ftn = page_holder_->GetPrimaryFrameTree().root();

  // Ensure that the pending frame policy is not set on the main frames, as it
  // is usually set on frames by their parent frames.
  if (prerender_root_ftn->pending_frame_policy() != blink::FramePolicy()) {
    return false;
  }

  if (primary_root_ftn->pending_frame_policy() != blink::FramePolicy()) {
    return false;
  }

  if (prerender_root_ftn->current_replication_state().frame_policy !=
      primary_root_ftn->current_replication_state().frame_policy) {
    return false;
  }

  return true;
}

bool PrerenderHost::AreInitialPrerenderNavigationParamsCompatibleWithNavigation(
    NavigationRequest& navigation_request) {
  // TODO(crbug.com/1181763): compare the rest of the navigation parameters. We
  // should introduce compile-time parameter checks as well, to ensure how new
  // fields should be compared for compatibility.

  // As the initial prerender navigation is a) limited to HTTP(s) URLs and b)
  // initiated by the PrerenderHost, we do not expect some navigation parameters
  // connected to certain navigation types to be set and the DCHECKS below
  // enforce that.
  // The parameters of the potential activation, however, are coming from the
  // renderer and we mostly don't have any guarantees what they are, so we
  // should not DCHECK them. Instead, by default we compare them with initial
  // prerender activation parameters and fail to activate when they differ.
  // Note: some of those parameters should be never set (or should be ignored)
  // for main-frame / HTTP(s) navigations, but we still compare them here as a
  // defence-in-depth measure.
  DCHECK(navigation_request.IsInPrimaryMainFrame());

  // Compare BeginNavigationParams.
  if (!AreBeginNavigationParamsCompatibleWithNavigation(
          navigation_request.begin_params())) {
    return false;
  }

  // Compare CommonNavigationParams.
  if (!AreCommonNavigationParamsCompatibleWithNavigation(
          navigation_request.common_params())) {
    return false;
  }

  return true;
}

bool PrerenderHost::AreBeginNavigationParamsCompatibleWithNavigation(
    const blink::mojom::BeginNavigationParams& potential_activation) {
  if (potential_activation.initiator_frame_token !=
      begin_params_->initiator_frame_token) {
    return false;
  }

  if (!AreHttpRequestHeadersCompatible(potential_activation.headers,
                                       begin_params_->headers)) {
    return false;
  }

  // Don't activate a prerendered page if the potential activation request
  // requires validation or bypass of the browser cache, as the prerendered page
  // is a kind of caches.
  // TODO(https://crbug.com/1213299): Instead of checking the load flags on
  // activation, we should cancel prerendering when the prerender initial
  // navigation has the flags.
  int cache_load_flags = net::LOAD_VALIDATE_CACHE | net::LOAD_BYPASS_CACHE |
                         net::LOAD_DISABLE_CACHE;
  if (potential_activation.load_flags & cache_load_flags) {
    return false;
  }
  if (potential_activation.load_flags != begin_params_->load_flags) {
    return false;
  }

  if (potential_activation.skip_service_worker !=
      begin_params_->skip_service_worker) {
    return false;
  }

  if (potential_activation.mixed_content_context_type !=
      begin_params_->mixed_content_context_type) {
    return false;
  }

  // Initial prerender navigation cannot be a form submission.
  DCHECK(!begin_params_->is_form_submission);
  if (potential_activation.is_form_submission !=
      begin_params_->is_form_submission) {
    return false;
  }

  if (potential_activation.searchable_form_url !=
      begin_params_->searchable_form_url) {
    return false;
  }

  if (potential_activation.searchable_form_encoding !=
      begin_params_->searchable_form_encoding) {
    return false;
  }

  // Trust token params can be set only on subframe navigations, so both values
  // should be null here.
  DCHECK(!begin_params_->trust_token_params);
  if (potential_activation.trust_token_params !=
      begin_params_->trust_token_params) {
    return false;
  }

  // Web bundle token cannot be set due because it is only set for child
  // frame navigations.
  DCHECK(!begin_params_->web_bundle_token);
  if (potential_activation.web_bundle_token) {
    return false;
  }

  // Don't require equality for request_context_type because link clicks
  // (HYPERLINK) should be allowed for activation, whereas prerender always has
  // type LOCATION.
  DCHECK_EQ(begin_params_->request_context_type,
            blink::mojom::RequestContextType::LOCATION);
  switch (potential_activation.request_context_type) {
    case blink::mojom::RequestContextType::HYPERLINK:
    case blink::mojom::RequestContextType::LOCATION:
      break;
    default:
      return false;
  }

  // Since impression should not be set, no need to compare contents.
  DCHECK(!begin_params_->impression);
  if (potential_activation.impression.has_value()) {
    return false;
  }

  // No need to test for devtools_initiator because this field is used for
  // tracking what triggered a network request, and prerender activation will
  // not use network requests.

  return true;
}

bool PrerenderHost::AreCommonNavigationParamsCompatibleWithNavigation(
    const blink::mojom::CommonNavigationParams& potential_activation) {
  // The CommonNavigationParams::url field is expected to be the same for both
  // initial and activation prerender navigations, as the PrerenderHost
  // selection would have already checked for matching values. Adding a DCHECK
  // here to be safe.
  if (attributes_.url_match_predicate) {
    DCHECK(
        attributes_.url_match_predicate.value().Run(potential_activation.url));
  } else {
    DCHECK_EQ(potential_activation.url, common_params_->url);
  }
  if (potential_activation.initiator_origin !=
      common_params_->initiator_origin) {
    return false;
  }

  if (potential_activation.transition != common_params_->transition) {
    return false;
  }

  DCHECK_EQ(common_params_->navigation_type,
            blink::mojom::NavigationType::DIFFERENT_DOCUMENT);
  if (potential_activation.navigation_type != common_params_->navigation_type) {
    return false;
  }

  // We don't check download_policy as it affects whether the download triggered
  // by the NavigationRequest is allowed to proceed (or logs metrics) and
  // doesn't affect the behaviour of the document created by a non-download
  // navigation after commit (e.g. it doesn't affect future downloads in child
  // frames). PrerenderNavigationThrottle has already ensured that the initial
  // prerendering navigation isn't a download and as prerendering activation
  // won't reach out to the network, it won't turn into a navigation as well.

  DCHECK(common_params_->base_url_for_data_url.is_empty());
  if (potential_activation.base_url_for_data_url !=
      common_params_->base_url_for_data_url) {
    return false;
  }

  // The previews_state is always set to NO_PREVIEWS in BeginNavigation and the
  // previews code was removed, so no need to compare it here as it's not used.
  // TODO(crbug.com/1232909): remove this previews_state.

  if (potential_activation.method != common_params_->method) {
    return false;
  }

  // Initial prerender navigation can't be a form submission.
  DCHECK(!common_params_->post_data);
  if (potential_activation.post_data != common_params_->post_data) {
    return false;
  }

  // No need to compare source_location, as it's only passed to the DevTools for
  // debugging purposes and does not impact the properties of the document
  // created by this navigation.

  DCHECK(!common_params_->started_from_context_menu);
  if (potential_activation.started_from_context_menu !=
      common_params_->started_from_context_menu) {
    return false;
  }

  // has_user_gesture doesn't affect any of the security properties of the
  // document created by navigation, so equality of the values is not required.
  // TODO(crbug.com/1232915): ensure that the user activation status is
  // propagated to the activated document.

  // text_fragment_token doesn't affect any of the security properties of the
  // document created by navigation, so equality of the values is not required.
  // TODO(crbug.com/1232919): ensure the activated document consumes
  // text_fragment_token and scrolls to the corresponding viewport.

  if (potential_activation.should_check_main_world_csp !=
      common_params_->should_check_main_world_csp) {
    return false;
  }

  if (potential_activation.initiator_origin_trial_features !=
      common_params_->initiator_origin_trial_features) {
    return false;
  }

  if (potential_activation.href_translate != common_params_->href_translate) {
    return false;
  }

  // Initial prerender navigation can't be a history navigation.
  DCHECK(!common_params_->is_history_navigation_in_new_child_frame);
  if (potential_activation.is_history_navigation_in_new_child_frame !=
      common_params_->is_history_navigation_in_new_child_frame) {
    return false;
  }

  // The spec mandates matching the referrer policy, and not the referrer URL
  // itself, so we only compare the referrer policy here. Referrer policy is a
  // more predictable value to match than referrer URL.
  // https://wicg.github.io/nav-speculation/prerendering.html#navigate-activation
  if (potential_activation.referrer->policy !=
      common_params_->referrer->policy) {
    return false;
  }

  if (potential_activation.request_destination !=
      common_params_->request_destination) {
    return false;
  }

  return true;
}

RenderFrameHostImpl* PrerenderHost::GetPrerenderedMainFrameHost() {
  DCHECK(page_holder_->frame_tree());
  DCHECK(page_holder_->frame_tree()->root()->current_frame_host());
  return page_holder_->frame_tree()->root()->current_frame_host();
}

FrameTree& PrerenderHost::GetPrerenderFrameTree() {
  DCHECK(page_holder_->frame_tree());
  return *page_holder_->frame_tree();
}

void PrerenderHost::RecordFinalStatus(base::PassKey<PrerenderHostRegistry>,
                                      FinalStatus status) {
  RecordFinalStatus(status, attributes_.initiator_ukm_id,
                    ukm::kInvalidSourceId);
}

void PrerenderHost::CreatePageHolder(WebContentsImpl& web_contents) {
  page_holder_ = std::make_unique<PageHolder>(web_contents);
  frame_tree_node_id_ =
      page_holder_->frame_tree()->root()->frame_tree_node_id();
}

PrerenderHost::LoadingOutcome PrerenderHost::WaitForLoadStopForTesting() {
  return page_holder_->WaitForLoadCompletionForTesting();  // IN-TEST
}

void PrerenderHost::RecordFinalStatus(FinalStatus status,
                                      ukm::SourceId initiator_ukm_id,
                                      ukm::SourceId prerendered_ukm_id) {
  DCHECK(!final_status_);
  final_status_ = status;
  RecordPrerenderHostFinalStatus(status, trigger_type(),
                                 embedder_histogram_suffix(), initiator_ukm_id,
                                 prerendered_ukm_id);
}

const GURL& PrerenderHost::GetInitialUrl() const {
  return attributes_.prerendering_url;
}

void PrerenderHost::AddObserver(Observer* observer) {
  observers_.AddObserver(observer);
}

void PrerenderHost::RemoveObserver(Observer* observer) {
  observers_.RemoveObserver(observer);
}

absl::optional<int64_t> PrerenderHost::GetInitialNavigationId() const {
  return initial_navigation_id_;
}

void PrerenderHost::SetInitialNavigation(NavigationRequest* navigation) {
  DCHECK(!initial_navigation_id_.has_value());
  initial_navigation_id_ = navigation->GetNavigationId();
  begin_params_ = navigation->begin_params().Clone();
  common_params_ = navigation->common_params().Clone();
}

bool PrerenderHost::IsUrlMatch(const GURL& url) const {
  // If the trigger defines its predicate, respect it.
  if (attributes_.url_match_predicate) {
    // Triggers are not allowed to treat a cross-origin url as a matched url. It
    // would cause security risks.
    if (!url::IsSameOriginWith(attributes_.prerendering_url, url))
      return false;
    return attributes_.url_match_predicate.value().Run(url);
  }
  return GetInitialUrl() == url;
}

void PrerenderHost::Cancel(FinalStatus status) {
  TRACE_EVENT("navigation", "PrerenderHost::Cancel", "final_status", status);
  // Already cancelled.
  if (final_status_)
    return;

  RenderFrameHostImpl* host = PrerenderHost::GetPrerenderedMainFrameHost();
  DCHECK(host);
  PrerenderHostRegistry* registry =
      host->delegate()->GetPrerenderHostRegistry();
  DCHECK(registry);
  registry->CancelHost(frame_tree_node_id_, status);
}

}  // namespace content


text:
namespace eeg_modelling {

::eeg_modelling::Status internal::StatusOrHelper::HandleInvalidStatusCtorArg() {
  ABSL_RAW_LOG(FATAL,
               "Status::OK is not a valid constructor argument to StatusOr<T>");
  // Workaround.
  return OkStatus();
}

::eeg_modelling::Status internal::StatusOrHelper::HandleNullObjectCtorArg() {
  ABSL_RAW_LOG(FATAL,
               "NULL is not a valid constructor argument to StatusOr<T*>");
  // Workaround.
  return OkStatus();
}

void internal::StatusOrHelper::Crash(const Status& status) {
  ABSL_RAW_LOG(FATAL,
               "Attempting to fetch value instead of handling error status");
}

}  // namespace eeg_modelling


text:
if (!String.prototype.startsWith) {
  String.prototype.startsWith = function(searchString, position) {
    return this.substr(position || 0, searchString.length) === searchString
  }
}


text:
module DeviseShibbolethAuthenticatable

  class Logger
    def self.send(message, logger = Rails.logger)
      if ::Devise.shibboleth_logger
        logger.add 0, "  \e[36mShibboleth:\e[0m #{message}"
      end
    end
  end

end


text:
<?php

namespace Admin\Model;

use Application\Model\Doctrine,
    DoctrineORMModule\Paginator\Adapter\DoctrinePaginator as DoctrineAdapter,
    Doctrine\ORM\Tools\Pagination\Paginator as ORMPaginator,
    User\Entity\Designer as DesignerEntity;

class Designer extends Doctrine {

  const LOGOS_DIR = 'public/images/designers/logo';
  const PROFILE_IMAGES_DIR = 'public/images/designers/profile';

  protected $designerEntity;
  protected $cdnHost;

  public function __construct($sm) {
    $envConfig = $sm->get('Config');
    $host = $envConfig['web']['host'];
    $this->cdnHost = $host;

    $em = $sm->get('doctrine.entitymanager.orm_default');
    parent::__construct($em);
  }

  public function getLogosDirectory() {
    return self::LOGOS_DIR;
  }

  public function getProfileImagesDirectory() {
    return self::PROFILE_IMAGES_DIR;
  }

  public function all() {
    return $this->entityManager->getRepository('User\Entity\Designer')->findAll();
  }

  public function allVisible($limit=0, $page=0) {
    if($page > 0) {
      $offset = (int) ($page-1)*$limit;
      return $this->entityManager->getRepository('User\Entity\Designer')->findBy( array('visible' => true), array('created_at' => 'ASC'), $limit, $offset );
    } else {
      $offset = (int) ($page-1)*$limit;
      return $this->entityManager->getRepository('User\Entity\Designer')->findBy( array('visible' => true), array('created_at' => 'ASC') );
    }
  }

  public function get($id) {
    return $this->entityManager->getRepository('User\Entity\Designer')->find($id); #->hydrate(\Doctrine\ORM\Query::HYDRATE_ARRAY);
  }

  public function getByIds($ids) {
    return $this->entityManager->getRepository('User\Entity\Designer')->findBy( array('id' => $ids) );
  }

  public function allSelect() {
    $designers = $this->entityManager->getRepository('User\Entity\Designer')->findAll();
    $data = array();
    foreach ($designers as $designer){
      $data[$designer->id] = $designer->name;
    }
    return $data;
  }

  public function getAdapter($column = 'sequence', $order = 'desc', $searchParams = array()) {
    $repository = $this->entityManager->getRepository('User\Entity\Designer');
    $query = $repository->createQueryBuilder('d');
    if ($column != '') {
      $query->add('orderBy', "d.$column $order");
    } else {
      $query->add('orderBy', "d.sequence desc");
    }
    if (isset($searchParams['query']) && $searchParams['query'] != '') {
      $searchTerm = $searchParams['query'];
      $orX = $query->expr()->orx();
      $orX->add( $query->expr()->like('d.name', $query->expr()->literal('%' . $searchTerm . '%')) );
      $orX->add( $query->expr()->like('d.description', $query->expr()->literal('%' . $searchTerm . '%')) );
      $query->andWhere($orX);
    }
    if (isset($searchParams['visible']) && $searchParams['visible'] != '') {
      $query->andWhere( $query->expr()->eq('d.visible', $searchParams['visible']) );
    }
    return new DoctrineAdapter(new ORMPaginator($query));
  }

  public function getNextSequence() {
    $repository = $this->entityManager->getRepository('User\Entity\Designer');
    $qb = $repository->createQueryBuilder('d');
    $qb->select( $qb->expr()->max('d.sequence') );
    $dql = $qb->getDql();
    $query = $this->entityManager->createQuery($dql);
    $nextSequence = $query->getSingleScalarResult() + 1;
    return $nextSequence;
  }

  public function getMaxSequenceByIds($ids) {
    $repository = $this->entityManager->getRepository('User\Entity\Designer');
    $qb = $repository->createQueryBuilder('d');
    $qb->select( $qb->expr()->max('d.sequence') );
    $qb->where( $qb->expr()->in('d.id', $ids) );
    $dql = $qb->getDql();
    $query = $this->entityManager->createQuery($dql);
    return $query->getSingleScalarResult();
  }

  public function create($data) {
    $designer = new DesignerEntity();
    foreach ($data as $key => $value) {
      $designer->$key = $value;
    }
    $designer->sequence = $this->getNextSequence();

    $this->entityManager->persist($designer);
    $this->entityManager->flush();
    return $designer;
  }

  public function setDesingerEntity($designer) {
    $this->designerEntity = $designer;
  }

  public function getDesingerEntity() {
    return $this->designerEntity;
  }

  public function getCdnHost() {
    return $this->cdnHost;
  }

  public function getServiceManager() {
    return $this->serviceManager;
  }

  public function getRepresentation() {
    $designer = $this->getDesingerEntity();
    $output['id'] = $designer->id;
    $output['name'] = $designer->name;
    $output['description'] = $designer->description;
    if ($designer->logo) {
      $output['logo'] = $this->getCdnHost() . '/images/designers/logo/' . $designer->logo . '?' . time();
    } else {
      $output['logo'] = '';
    }
    if ($designer->profile_pic) {
      $output['profile_picture'] = $this->getCdnHost() . '/images/designers/profile/' . $designer->profile_pic . '?' . time();
    } else {
      $output['profile_picture'] = '';
    }
    return $output;
  }

  public function removeLogo($id) {
    $designer = $this->get($id);
    if($designer instanceof DesignerEntity) {
      $logosDirectory = $this->getLogosDirectory();
      $logoImageFile = $designer->logo;
      $logoTargetPath = "$logosDirectory/$logoImageFile";
      if(file_exists($logoTargetPath)) {
        unlink($logoTargetPath);
      }
      $designer->logo = null;
      $this->save($designer);
    }
  }

  public function removeProfileImage($id) {
    $designer = $this->get($id);
    if($designer instanceof DesignerEntity) {
      $profileImagesDirectory = $this->getProfileImagesDirectory();
      $profileImageFile = $designer->profile_pic;
      $profileImageTargetPath = "$profileImagesDirectory/$profileImageFile";
      if(file_exists($profileImageTargetPath)) {
        unlink($profileImageTargetPath);
      }
      $designer->profile_pic = null;
      $this->save($designer);
    }
  }

}


text:
import * as _ from 'lodash';
import * as nock from 'nock';
import 'should';

import { common } from '@bitgo/sdk-core';

import { TestBitGo } from '@bitgo/sdk-test';
import { BitGo } from '../../src/bitgo';

describe('Keychains', function v2keychains() {
  describe('Update Password', function updatePassword() {
    let bitgo;
    let keychains;
    let bgUrl;

    before(function beforeDescribe() {
      nock.pendingMocks().should.be.empty();

      bitgo = TestBitGo.decorate(BitGo, { env: 'mock' });
      bitgo.initializeTestVars();
      bitgo.setValidate(false);
      keychains = bitgo.keychains();

      bgUrl = common.Environments[bitgo.getEnv()].uri;
    });

    it('should fail to update the password', async function () {
      await keychains.updatePassword({ newPassword: '5678' })
        .should.be.rejectedWith('Missing parameter: oldPassword');

      await keychains.updatePassword({ oldPassword: 1234, newPassword: '5678' })
        .should.be.rejectedWith('Expecting parameter string: oldPassword but found number');

      await keychains.updatePassword({ oldPassword: '1234' })
        .should.be.rejectedWith('Missing parameter: newPassword');

      await keychains.updatePassword({ oldPassword: '1234', newPassword: 5678 })
        .should.be.rejectedWith('Expecting parameter string: newPassword but found number');
    });

    it('successful password update', async function () {
      const oldPassword = 'oldPassword';
      const newPassword = 'newPassword';
      const otherPassword = 'otherPassword';

      nock(bgUrl)
        .post('/api/v1/user/encrypted')
        .reply(200, {
          keychains: {
            xpub1: bitgo.encrypt({ input: 'xprv1', password: oldPassword }),
            xpub2: bitgo.encrypt({ input: 'xprv2', password: otherPassword }),
          },
          version: 1,
        });

      const result = await keychains.updatePassword({ oldPassword: oldPassword, newPassword: newPassword });
      _.forOwn(result.keychains, function (encryptedXprv, xpub) {
        xpub.should.startWith('xpub');
        try {
          const decryptedPrv = bitgo.decrypt({ input: encryptedXprv, password: newPassword });
          decryptedPrv.should.startWith('xprv');
        } catch (e) {
          // the decryption didn't work because of the wrong password, this is one of the keychains that didn't match
          // the old password
          e.message.should.startWith('password error');
        }
      });
      result.should.hasOwnProperty('version');
    });
  });

  after(function afterKeychains() {
    nock.pendingMocks().should.be.empty();
  });
});


text:
using Cucumis.Specifications.Support;
using TechTalk.SpecFlow;

namespace Cucumis.Specifications.Steps
{
    [Binding]
    internal sealed class InheritanceSteps
    {
        [When("entrypoint is invoked on inherited automation class")]
        public void WhenEntrypointIsInvokedOnInheritedAutomationClass()
        {
            // The InheritedGardener is defined within the test namespace
            new InheritedGardener().PlantGherkin();
        }

        [When("entrypoint is invoked through an inherited automation class")]
        public void WhenEntrypointIsInvokedThroughAnInheritedAutomationClass()
        {
            // The InheritedGardener is so much more efficient
            new InheritedGardener().PlantGherkinAndWaterIt();
        }

        [When("same method is invoked with different concrete types")]
        public void WhenSameMethodIsInvokedWithDifferentConcreteTypes()
        {
            var gardener = new Gardener();
            gardener.HarvestGherkin(new Gherkin());
            gardener.HarvestGherkin(new PickyGherkin());
        }

        [When("an instance method is invoked from its base")]
        public void WhenAnInstanceMethodIsInvokedFromItsBase()
        {
            // The Bloom method is only defined on the base
            new Melothria().Bloom();
        }

        [When("a base method is called from a far off generations")]
        public void WhenABaseMethodIsCalledFromFarOffGenerations()
        {
            Gherkin gherkin = new StubbornGherkin();
            gherkin.CutVine();
        }

        [When("an inherited instance method is invoked indirectly")]
        public void WhenAnInheritedInstanceMethodIsInvokedIndirectly()
        {
            PickyGherkin gherkin = new PickyGherkin();
            PrepareAndCutVine(gherkin);
        }

        private static void PrepareAndCutVine(Gherkin gherkin)
        {
            CutTheVine(gherkin);
        }

        private static void CutTheVine(Gherkin gherkin)
        {
            gherkin.CutVine();
        }
    }
}


text:
from benchmark.celery import app
from benchmark.manager import benchmark_manager
from stream_framework.utils import get_metrics_instance
from benchmark import feeds

@app.task(bind=True, ignore_result=True)
def add_activities(self, social_model, user_activity_tuples):
    for user_id, activity in user_activity_tuples:
        # inserts into cassandra and creates the fanout tasks
        follower_ids = social_model.get_follower_ids(user_id)
        benchmark_manager.add_user_activity(user_id, activity, follower_ids)

@app.task(bind=True, ignore_result=True)
def read_feed_pages(self, social_model, user_ids):
    for user_id in user_ids:
        browse_depth = social_model.get_browse_depth(user_id)
        feed_instance = feeds.TimelineFeed(user_id)
        metrics = get_metrics_instance()
        # browse x pages deep
        for page in range(browse_depth):
            # track the time every read takes
            with metrics.feed_reads_timer(feed_instance.__class__):
                activities = feed_instance[:25]
                if activities and len(activities) == 25:
                    last_id = activities[-1]
                    feed_instance.filter(id__lte=last_id)
                else:
                    break

@app.task(bind=True, ignore_result=True)
def follow_users(self, social_model, follows):
    # user a follows user b. since we're already in a task, run with
    # async=False
    for user_id, target_user_id in follows:
        benchmark_manager.follow_user(user_id, target_user_id, async=False)


text:
package org.keycloak.federation.ldap.mappers;

import java.util.ArrayList;
import java.util.List;

import org.keycloak.mappers.MapperConfigValidationException;
import org.keycloak.mappers.UserFederationMapper;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.LDAPConstants;
import org.keycloak.models.RealmModel;
import org.keycloak.models.UserFederationMapperModel;
import org.keycloak.provider.ProviderConfigProperty;

/**
 * @author Marek Posolda
 */
public class FullNameLDAPFederationMapperFactory extends AbstractLDAPFederationMapperFactory {

    public static final String PROVIDER_ID =  "full-name-ldap-mapper";

    protected static final List<ProviderConfigProperty> configProperties = new ArrayList<ProviderConfigProperty>();

    static {
        ProviderConfigProperty userModelAttribute = createConfigProperty(FullNameLDAPFederationMapper.LDAP_FULL_NAME_ATTRIBUTE, "LDAP Full Name Attribute",
                "Name of LDAP attribute, which contains fullName of user. In most cases it will be 'cn' ", ProviderConfigProperty.STRING_TYPE, LDAPConstants.CN);
        configProperties.add(userModelAttribute);

        ProviderConfigProperty readOnly = createConfigProperty(UserAttributeLDAPFederationMapper.READ_ONLY, "Read Only",
                "For Read-only is data imported from LDAP to Keycloak DB, but it's not saved back to LDAP when user is updated in Keycloak.", ProviderConfigProperty.BOOLEAN_TYPE, "false");
        configProperties.add(readOnly);
    }

    @Override
    public String getHelpText() {
        return "Used to map full-name of user from single attribute in LDAP (usually 'cn' attribute) to firstName and lastName attributes of UserModel in Keycloak DB";
    }

    @Override
    public String getDisplayCategory() {
        return ATTRIBUTE_MAPPER_CATEGORY;
    }

    @Override
    public String getDisplayType() {
        return "Full Name";
    }

    @Override
    public List<ProviderConfigProperty> getConfigProperties() {
        return configProperties;
    }

    @Override
    public String getId() {
        return PROVIDER_ID;
    }

    @Override
    public void validateConfig(UserFederationMapperModel mapperModel) throws MapperConfigValidationException {
        checkMandatoryConfigAttribute(FullNameLDAPFederationMapper.LDAP_FULL_NAME_ATTRIBUTE, "LDAP Full Name Attribute", mapperModel);
    }

    @Override
    public UserFederationMapper create(KeycloakSession session) {
        return new FullNameLDAPFederationMapper();
    }
}


text:
import { Disposable } from 'vs/base/common/lifecycle';
import { join } from 'vs/base/common/path';
import * as pfs from 'vs/base/node/pfs';
import { INativeEnvironmentService } from 'vs/platform/environment/common/environment';
import { DidUninstallExtensionEvent, IExtensionManagementService, InstallExtensionResult } from 'vs/platform/extensionManagement/common/extensionManagement';
import { MANIFEST_CACHE_FOLDER, USER_MANIFEST_CACHE_FILE } from 'vs/platform/extensions/common/extensions';

export class ExtensionsManifestCache extends Disposable {

	private extensionsManifestCache = join(this.environmentService.userDataPath, MANIFEST_CACHE_FOLDER, USER_MANIFEST_CACHE_FILE);

	constructor(
		private readonly environmentService: INativeEnvironmentService,
		extensionsManagementService: IExtensionManagementService
	) {
		super();
		this._register(extensionsManagementService.onDidInstallExtensions(e => this.onDidInstallExtensions(e)));
		this._register(extensionsManagementService.onDidUninstallExtension(e => this.onDidUnInstallExtension(e)));
	}

	private onDidInstallExtensions(results: readonly InstallExtensionResult[]): void {
		if (results.some(r => !!r.local)) {
			this.invalidate();
		}
	}

	private onDidUnInstallExtension(e: DidUninstallExtensionEvent): void {
		if (!e.error) {
			this.invalidate();
		}
	}

	invalidate(): void {
		pfs.Promises.rm(this.extensionsManifestCache, pfs.RimRafMode.MOVE).then(() => { }, () => { });
	}
}