text:
import {bookmarkFileToLoad} from "./hillview";
import {DatasetView} from "./datasetView";
import {InitialObject} from "./initialObject";
import {LoadView} from "./loadView";
import {FullPage, PageTitle} from "./ui/fullPage";
import {ContextMenu} from "./ui/menu";
import {IHtmlElement, removeAllChildren} from "./ui/ui";
import {UIConfig} from "./javaBridge";

/**
 * The toplevel class implements the web page structure for Hillview.
 * The web page has a load menu, a list of tabs, and the view of the current tab.
 * Each tab corresponds to one Dataset.
 */
export class HillviewToplevel implements IHtmlElement {
    private readonly topLevel: HTMLElement;
    public readonly datasets: DatasetView[];
    private readonly strip: HTMLDivElement;
    private readonly tabs: HTMLElement[];
    private readonly content: HTMLDivElement;
    protected datasetCounter: number;
    protected current: DatasetView | null;
    public uiconfig: UIConfig;

    public static readonly instance = new HillviewToplevel();

    private constructor() {
        this.datasets = [];
        this.datasetCounter = 0;
        this.current = null;
        this.topLevel = document.createElement("div");
        const page = new FullPage(0, new PageTitle("Load data", ""),
            null, null);

        this.topLevel.appendChild(page.getHTMLRepresentation());
        const loadPage = new LoadView(InitialObject.instance, page, bookmarkFileToLoad);
        page.setDataView(loadPage);
        page.getTitleElement().ondblclick = () => loadPage.toggleAdvanced();

        const tabStrip = document.createElement("div");
        this.topLevel.appendChild(document.createElement("hr"));
        this.topLevel.appendChild(tabStrip);
        this.strip = document.createElement("div");
        this.strip.className = "tabs-strip";
        this.strip.style.display = "flex";
        this.strip.style.width = "100%";
        this.strip.style.flexDirection = "row";
        this.strip.style.flexWrap = "nowrap";
        this.strip.style.alignItems = "center";

        tabStrip.appendChild(this.strip);
        this.tabs = [];
        // default configuration
        this.uiconfig = {};
        this.content = document.createElement("div");
        this.topLevel.appendChild(this.content);
    }

    public setUIConfig(uiconfig: UIConfig): void {
        this.uiconfig = uiconfig;
    }

    public getHTMLRepresentation(): HTMLElement {
        return this.topLevel;
    }

    public addDataset(dataset: DatasetView, menu: ContextMenu): void {
        // tab names never change once tabs are created.  They are also unique.
        // The tab name is not the dataset name; the dataset name is displayed in the tab.
        const tabName = "tab" + this.datasetCounter++;
        this.datasets.push(dataset);

        const tab = document.createElement("table");
        tab.className = "tab";
        tab.id = tabName;
        const row = tab.insertRow();

        this.strip.appendChild(tab);
        this.tabs.push(tab);
        let cell = row.insertCell();
        cell.textContent = dataset.name;
        cell.title = dataset.name + "\nRight-click opens a menu. Click to edit.";
        cell.className = "dataset-name";
        cell.oncontextmenu = (e) => menu.showAtMouse(e);
        cell.onclick = () => this.select(tabName);

        const close = document.createElement("span");
        close.className = "close";
        close.innerHTML = "×";
        close.setAttribute("float", "right");
        close.onclick = () => this.remove(tabName);
        close.title = "Close this dataset.";

        cell = row.insertCell();
        cell.appendChild(close);
        this.select(tabName);
    }

    public index(tabName: string): number {
        return this.tabs.map((d) => d.id).lastIndexOf(tabName);
    }

    public remove(tabName: string): void {
        const index = this.index(tabName);
        if (index < 0)
            return;
        this.strip.removeChild(this.tabs[index]);
        this.datasets.splice(index, 1);
        this.tabs.splice(index, 1);
        if (this.tabs.length >= 1) {
            this.select(this.tabs[0].id);
        } else {
            removeAllChildren(this.content);
            this.current = null;
        }
    }

    /**
     * Select the tab with the specified name (a string like tabXXX).
     * @param tabName    Name of tab to select.
     */
    public select(tabName: string): void {
        const index = this.index(tabName);
        if (index < 0)
            return;
        for (let i = 0; i < this.strip.childElementCount; i++) {
            const child: HTMLElement = this.tabs[i];
            const cell = child.querySelector<HTMLElement>(".dataset-name")!;
            if (i !== index) {
                child.classList.remove("current");
                cell.contentEditable = "false";
            } else {
                child.classList.add("current");
                cell.contentEditable = "true";
                cell.onblur = () => dataset.rename(cell.textContent);
                cell.onkeydown = (e) => {
                    if (e.key == "Enter") {
                        cell.blur();
                    }
                };
            }
        }

        removeAllChildren(this.content);
        const dataset = this.datasets[index];
        this.current = dataset;
        this.content.appendChild(dataset.getHTMLRepresentation());
        dataset.resize();
    }

    public resize(): void {
        if (this.current != null) {
            this.current.resize();
        }
    }
}

export function createHillview(): void {
    const top = document.getElementById("top");
    top!.appendChild(HillviewToplevel.instance.getHTMLRepresentation());
    window.addEventListener("resize",  () => HillviewToplevel.instance.resize());
}


text:
let moment = require("moment");
let sprintf = require("sprintf-js").sprintf;
let chalk = require("chalk");
let emoji = require("node-emoji");
let winston = require("winston");
let util = require("./utility.js");
let cli = require("./cli.js");
let twitter = require("twitter");

const exec = require("child_process").exec;

let getLogger = (path) => {
  let settings = {
    transports: [
      new winston.transports.File({ filename: path, json: true})
    ],
    exceptionHandlers: [
      new winston.transports.File({ filename: path, json: true})
    ]
  };

  return new (winston.Logger)(settings);
};

let core = {};

core.display = (data)  => {
  let name, channel;

  if (util.users[data.user]) {
    name = chalk[util.users[data.user].color](util.users[data.user].name);
  } else {
    if (typeof data.user == "string") {
      name = chalk.white(data.user);
    } else {
      name = chalk.white("-");
    }
  }

  if (util.channels[data.channel]) {
    channel = "#" + chalk[util.channels[data.channel].color](util.channels[data.channel].name);
  } else {
    if (typeof data.channel == "string") {
      channel = chalk.white(data.channel);
    } else {
      channel = chalk.white("-");
    }
  }

  data.lines.forEach((line) => {
    let l;

    l = emoji.emojify(line);
    l = util.replaceSlackId(l);
    l = util.decolateText(l);

    let dateFormat = data.time.format("YYYY-MM-DD HH:mm:ss");

    if (util.theme.date) {
      dateFormat = chalk[util.theme.date](dateFormat);
    }

    console.log(
      "%s | %s | %s | %s",
      dateFormat,
      sprintf("%30s", channel),
      sprintf("%28s", name),
      l
    );

    name = chalk.white("|>");
  });
};

core.start = async (commander) => {
  const options = commander.opts();

  let logger;
  let token = options.token;

  if (options.debug) {
    logger = getLogger(options.debug);
  }

  if (options.settings) {
    util.parseSettingFile(options.settings);

    if (util.token) {
      token = util.token;
    }
  }

  const {RTMClient} = require("@slack/client");
  const rtm = new RTMClient(token, {logLevel: "error"});

  let colors = ["red", "green", "yellow", "blue", "magenta", "cyan", "white", "redBright", "greenBright", "yellowBright", "blueBright", "magentaBright", "cyanBright", "whiteBright"];

  util.channels = {};
  util.users = {};
  util.bots = {};
  util.startUp = false;

  if (util.twitter) {
    let lastestTwId=1;
    let twitterClient = new twitter(util.twitter);

    let collectTweets = () => {
      twitterClient.get("statuses/home_timeline", {count:30, since_id:lastestTwId}, (error,tweets) => {

        if (error) {
          console.log(error);
          return;
        }

        if (Array.isArray(tweets)) {

          if (tweets.length > 0) {
            lastestTwId = tweets[0].id;
            tweets.reverse();
          }

          tweets.forEach((tweet) => {
            let data = {
              bufferKey: "twitter",
              lines: tweet.text.split(/\n/),
              time: moment(tweet.created_at),
              channel: "[Twitter]",
              user: tweet.user.screen_name
            };

            core.display(data);
          });
        }
      });
    };

    setInterval(collectTweets, 1000 * 60);
  }

  rtm.on("authenticated", (rtmStartData) => {
    if (!util.startUp) {
      console.log(
        `Logged in as ${chalk.bold(rtmStartData.self.name)} of team ${chalk.green.bold(rtmStartData.team.name)}, but not yet connected to a channel`
      );

      util.startUp = true;
    }
  });

  rtm.on("message", (message) => {
    let time = moment(message.ts * 1000);
    let text = util.parseText(message);

    if (message.subtype) {
      switch(message.subtype) {
      case "message_deleted":
      case "message_changed":
        return;
      case "file_share":
        break;
      case "message_replied":
        return;
      case "channel_join":
        break;
      case "channel_leave":
        break;
      case "group_leave":
        break;
      case "reply_broadcast":
        return;
      case "file_comment":
        break;
      case "bot_message":
        break;
      case "pinned_item":
        break;
      case "slackbot_response":
        break;
      case "file_mention":
        break;
      default:
        if (options.debug) {
          logger.error(message);
        }
        return;
      }
    }

    if (text == "") {
      if (Object.prototype.hasOwnProperty.call(message, "attachments")) {
        text = message.attachments[0].text;
      } else {
        text = "";
      }
    }

    let lines;

    if (typeof(text) == "string") {
      lines = text.split(/\r\n|\r|\n/);
    } else {
      lines = [""];

      if (options.debug) {
        logger.error(message);
      }
    }

    if (lines.length > 8) {
      lines = lines.slice(0, 5);
      lines.push("--- snip ---");
    }

    // Display only specific users. (-u option)
    if (options.user) {
      let messageUser = (util.users[message.user]) ? util.users[message.user].name : "-";
      if (options.user != messageUser) {
        return;
      }
    }

    // buffering
    let data = {
      bufferKey: (util.channels[message.channel]) ? "#" + util.channels[message.channel].name : "-",
      lines: lines,
      time: time,
      channel: message.channel,
      user: message.user
    };

    util.addMessageBuffer(data);
    core.display(data);

    // hook
    if (options.hook) {
      if (util.hook) {
        exec(util.hook, (err) => {
          if (err) {
            console.log(err);
          }
        });
      }
    }

    // hooks
    let hooks = util.hasHooks(message);

    hooks.forEach((hook) => {
      exec(hook, (err) => {
        if (err) {
          console.log(err);
        }
      });
    });
  });

  // Setup
  const {WebClient} = require("@slack/client");
  const web = new WebClient(token, {logLevel: "error"});

  let response = await web.conversations.list({
    limit:1000,
    types: "public_channel,private_channel"
  });

  response.channels.forEach((v, i) => {
    v.color = colors[i % colors.length];
    util.channels[v.id] = v;
  });

  while(response.response_metadata.next_cursor != "") {
    response = await web.conversations.list({
      limit: 1000,
      types: "public_channel,private_channel",
      cursor: response.response_metadata.next_cursor
    });

    response.channels.forEach((v, i) => {
      v.color = colors[i % colors.length];
      util.channels[v.id] = v;
    });
  }

  response = await web.users.list();
  while(response.response_metadata.next_cursor != "") {
    response.members.forEach((v, i) => {
      v.color = colors[i % colors.length];
      util.users[v.id] = v;
    });

    response = await web.users.list({
      limit: 1000,
      cursor: response.response_metadata.next_cursor
    });
  }

  // complete
  rtm.start();
};

// Declare cli-handler
function handler() {}

handler.prototype.recent = function(args, fn) {
  if (util.buffer[args[0]]) {
    let messageBuffer = util.buffer[args[0]];

    let label = {
      lines: [`--- Show buffer (#${args[0]}) ---`],
      time: moment()
    };

    core.display(label);

    messageBuffer.forEach((data) => {
      core.display(data);
    });

    label.lines = ["--- finish ---"];
    core.display(label);

  }

  fn(null, args);
};

handler.prototype.echo = function(args, fn) {
  fn(null, args);
};

handler.prototype.exit = function(args, fn) {
  fn(null, args);
  this.emit("close");
};

(new cli(new handler())).run();

module.exports = core;

redpajama_set_name: RedPajamaGithub


text:
package net.sf.katta.util;

import java.lang.reflect.Method;
import java.util.Arrays;

import net.sf.katta.client.Client;
import net.sf.katta.client.ClientResult;
import net.sf.katta.client.INodeSelectionPolicy;

import org.apache.log4j.Logger;

/**
 * The front end for a test server that just sleeps for a while then returns
 * nothing. Used for testing.
 */
public class SleepClient implements ISleepClient {

  protected final static Logger LOG = Logger.getLogger(SleepClient.class);

  private Client _client;

  public SleepClient(final INodeSelectionPolicy nodeSelectionPolicy) {
    _client = new Client(ISleepServer.class, nodeSelectionPolicy);
  }

  public SleepClient() {
    _client = new Client(ISleepServer.class);
  }

  public SleepClient(final ZkConfiguration config) {
    _client = new Client(ISleepServer.class, config);
  }

  public SleepClient(final INodeSelectionPolicy policy, final ZkConfiguration config) {
    _client = new Client(ISleepServer.class, policy, config);
  }

  public SleepClient(final INodeSelectionPolicy policy, final ZkConfiguration config,
          final ClientConfiguration clientConfiguration) {
    _client = new Client(ISleepServer.class, policy, config, clientConfiguration);
  }

  private static final Method SLEEP_METHOD;
  private static final int SLEEP_METHOD_SHARD_ARG_IDX = 2;
  static {
    try {
      SLEEP_METHOD = ISleepServer.class.getMethod("sleep", new Class[] { Long.TYPE, Integer.TYPE, String[].class });
    } catch (NoSuchMethodException e) {
      throw new RuntimeException("Could not find method sleep() in ISleepServer!");
    }
  }

  public int sleep(final long msec) throws KattaException {
    return sleepShards(msec, 0, null);
  }

  public int sleep(final long msec, final int delta) throws KattaException {
    return sleepShards(msec, delta, null);
  }

  public int sleepShards(final long msec, final String[] shards) throws KattaException {
    return sleepShards(msec, 0, shards);
  }

  public int sleepShards(final long msec, final int delta, final String[] shards) throws KattaException {
    ClientResult<Integer> results = _client.broadcastToShards(msec + delta + 3000, true, SLEEP_METHOD,
            SLEEP_METHOD_SHARD_ARG_IDX, shards != null ? Arrays.asList(shards) : null, msec, delta, null);
    if (results.isError()) {
      throw results.getKattaException();
    }
    int totalShards = 0;
    for (int numShards : results.getResults()) {
      totalShards += numShards;
    }
    return totalShards;
  }

  public int sleepIndices(final long msec, final String[] indices) throws KattaException {
    return sleepIndices(msec, 0, indices);
  }

  public int sleepIndices(final long msec, final int delta, final String[] indices) throws KattaException {
    ClientResult<Integer> results = _client.broadcastToIndices(msec + delta + 3000, true, SLEEP_METHOD,
            SLEEP_METHOD_SHARD_ARG_IDX, indices, msec, delta, null);
    if (results.isError()) {
      throw results.getKattaException();
    }
    int totalShards = 0;
    for (int numShards : results.getResults()) {
      totalShards += numShards;
    }
    return totalShards;
  }

  public Client getClient() {
    return _client;
  }

  public void close() {
    _client.close();
  }

}


text:
layout: post
title: "Curation #1 : L'indigeste"
date: 2017-09-25
categories: _post
author: irene_dunyach
published: true
image:
  teaser: post3-thumb.jpg
---

Pour démarrer les curations du digeste, il semblait opportun de parler de digestion. (Je m'excuse par avance car tous les liens sélectionnés ici sont en anglais)

Je digère mal.

En règle générale, toute transition chez moi se fait difficilement. Je digère mal les tomates, les poivrons, les féculents, les transformations, les annonces soudaines, les changements, les critiques, les échecs.

Ce qui m'amène à établir un lien certain entre mon transit intestinal et ma capacité à absorber des renseignements. J'ai l'impression de sentir dans mon cerveau la trajectoire d'une nouvelle information qui se ferait peu à peu broyer, décortiquer, assimiler par mon esprit jusqu'à me rendre capable d'en faire sens. Ce n'est pas un processus immédiat et les rouages sont comme ceux de mon système digestif : lents, bruyants, souvent partiellement inefficaces.

Il ne s'agit pas seulement de noter que notre esprit a un impact direct et certain sur nos habitudes alimentaires, mais de véritablement établir une similarité dans le process mental et intestinal. De là naissent quelques questions :

Si on peut dire "loin des yeux, loin du cœur", peut-on ajouter, loin des yeux, loin de l'estomac ?

Peut-on véritablement digérer une rupture amoureuse ?

Quelle est votre comfort-food mentale ?

Mes tripes sont-elles un second cerveau viable ? Peut-on espérer un jour réfléchir entièrement par l'estomac ? Et si oui : quelles nourritures vont influencer au mieux mon nouvel espace mental ?

Et enfin : si nos intestins produisent des déchets, quels sont les reliquats de nos digestions mentales ?

À bientôt.

Illustrations : Irène Dune


text:
module CycleAnalyticsParams
  extend ActiveSupport::Concern

  def cycle_analytics_project_params
    return {} unless params[:cycle_analytics].present?

    params[:cycle_analytics].permit(:start_date, :created_after, :created_before, :branch_name)
  end

  def cycle_analytics_group_params
    return {} unless params.present?

    params.permit(:group_id, :start_date, :created_after, :created_before, project_ids: [])
  end

  def options(params)
    @options ||= { from: start_date(params), current_user: current_user }.merge(date_range(params))
  end

  def start_date(params)
    case params[:start_date]
    when '7'
      7.days.ago
    when '30'
      30.days.ago
    else
      90.days.ago
    end
  end

  def date_range(params)
    {}.tap do |date_range_params|
      date_range_params[:from] = to_utc_time(params[:created_after]).beginning_of_day if params[:created_after]
      date_range_params[:to] = to_utc_time(params[:created_before]).end_of_day if params[:created_before]
    end.compact
  end

  def to_utc_time(field)
    date = field.is_a?(Date) || field.is_a?(Time) ? field : Date.parse(field)
    date.to_time.utc
  end
end

CycleAnalyticsParams.prepend_if_ee('EE::CycleAnalyticsParams')


text:
package org.apache.beam.sdk.io.gcp.firestore;

import static java.util.Objects.requireNonNull;

import com.google.cloud.firestore.v1.stub.FirestoreStub;
import com.google.firestore.v1.BatchWriteRequest;
import com.google.firestore.v1.BatchWriteResponse;
import com.google.firestore.v1.DatabaseRootName;
import com.google.firestore.v1.Write;
import com.google.firestore.v1.WriteResult;
import com.google.rpc.Code;
import com.google.rpc.Status;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.PriorityQueue;
import java.util.Queue;
import java.util.stream.Collectors;
import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
import org.apache.beam.sdk.io.gcp.firestore.FirestoreDoFn.ExplicitlyWindowedFirestoreDoFn;
import org.apache.beam.sdk.io.gcp.firestore.FirestoreV1.FailedWritesException;
import org.apache.beam.sdk.io.gcp.firestore.FirestoreV1.WriteFailure;
import org.apache.beam.sdk.io.gcp.firestore.FirestoreV1.WriteSuccessSummary;
import org.apache.beam.sdk.io.gcp.firestore.FirestoreV1RpcAttemptContexts.HasRpcAttemptContext;
import org.apache.beam.sdk.io.gcp.firestore.FirestoreV1RpcAttemptContexts.V1FnRpcAttemptContext;
import org.apache.beam.sdk.io.gcp.firestore.RpcQos.RpcAttempt.Context;
import org.apache.beam.sdk.io.gcp.firestore.RpcQos.RpcWriteAttempt;
import org.apache.beam.sdk.io.gcp.firestore.RpcQos.RpcWriteAttempt.Element;
import org.apache.beam.sdk.io.gcp.firestore.RpcQos.RpcWriteAttempt.FlushBuffer;
import org.apache.beam.sdk.metrics.Counter;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.BackOffUtils;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.joda.time.Duration;
import org.joda.time.Instant;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * A collection of {@link org.apache.beam.sdk.transforms.DoFn DoFn}s for each of the supported write
 * RPC methods from the Cloud Firestore V1 API.
 */
final class FirestoreV1WriteFn {

  static final class BatchWriteFnWithSummary extends BaseBatchWriteFn<WriteSuccessSummary> {
    BatchWriteFnWithSummary(
        JodaClock clock,
        FirestoreStatefulComponentFactory firestoreStatefulComponentFactory,
        RpcQosOptions rpcQosOptions,
        CounterFactory counterFactory) {
      super(clock, firestoreStatefulComponentFactory, rpcQosOptions, counterFactory);
    }

    @Override
    void handleWriteFailures(
        ContextAdapter<WriteSuccessSummary> context,
        Instant timestamp,
        List<KV<WriteFailure, BoundedWindow>> writeFailures,
        Runnable logMessage) {
      throw new FailedWritesException(
          writeFailures.stream().map(KV::getKey).collect(Collectors.toList()));
    }

    @Override
    void handleWriteSummary(
        ContextAdapter<WriteSuccessSummary> context,
        Instant timestamp,
        KV<WriteSuccessSummary, BoundedWindow> tuple,
        Runnable logMessage) {
      logMessage.run();
      context.output(tuple.getKey(), timestamp, tuple.getValue());
    }
  }

  static final class BatchWriteFnWithDeadLetterQueue extends BaseBatchWriteFn<WriteFailure> {
    BatchWriteFnWithDeadLetterQueue(
        JodaClock clock,
        FirestoreStatefulComponentFactory firestoreStatefulComponentFactory,
        RpcQosOptions rpcQosOptions,
        CounterFactory counterFactory) {
      super(clock, firestoreStatefulComponentFactory, rpcQosOptions, counterFactory);
    }

    @Override
    void handleWriteFailures(
        ContextAdapter<WriteFailure> context,
        Instant timestamp,
        List<KV<WriteFailure, BoundedWindow>> writeFailures,
        Runnable logMessage) {
      logMessage.run();
      for (KV<WriteFailure, BoundedWindow> kv : writeFailures) {
        context.output(kv.getKey(), timestamp, kv.getValue());
      }
    }

    @Override
    void handleWriteSummary(
        ContextAdapter<WriteFailure> context,
        Instant timestamp,
        KV<WriteSuccessSummary, BoundedWindow> tuple,
        Runnable logMessage) {
      logMessage.run();
    }
  }

  /**
   * {@link DoFn} for Firestore V1 {@link BatchWriteRequest}s.
   *
   *
Writes will be enqueued to be sent at a potentially later time when more writes are
   * available. This Fn attempts to maximize throughput while maintaining a high request success
   * rate.
   *
   *
All request quality-of-service is managed via the instance of {@link RpcQos} associated with
   * the lifecycle of this Fn.
   */
  abstract static class BaseBatchWriteFn<OutT> extends ExplicitlyWindowedFirestoreDoFn<Write, OutT>
      implements HasRpcAttemptContext {
    private static final Logger LOG =
        LoggerFactory.getLogger(
            FirestoreV1RpcAttemptContexts.V1FnRpcAttemptContext.BatchWrite.getNamespace());
    private final JodaClock clock;
    private final FirestoreStatefulComponentFactory firestoreStatefulComponentFactory;
    private final RpcQosOptions rpcQosOptions;
    private final CounterFactory counterFactory;
    private final V1FnRpcAttemptContext rpcAttemptContext;

    // transient running state information, not important to any possible checkpointing
    //  worker scoped state
    private transient RpcQos rpcQos;
    private transient Counter writesSuccessful;
    private transient Counter writesFailedRetryable;
    private transient Counter writesFailedNonRetryable;
    //  bundle scoped state
    private transient FirestoreStub firestoreStub;
    private transient DatabaseRootName databaseRootName;

    @VisibleForTesting
    transient Queue<@NonNull WriteElement> writes = new PriorityQueue<>(WriteElement.COMPARATOR);

    @VisibleForTesting transient int queueNextEntryPriority = 0;

    @SuppressWarnings(
        "initialization.fields.uninitialized") // allow transient fields to be managed by component
    // lifecycle
    BaseBatchWriteFn(
        JodaClock clock,
        FirestoreStatefulComponentFactory firestoreStatefulComponentFactory,
        RpcQosOptions rpcQosOptions,
        CounterFactory counterFactory) {
      this.clock = clock;
      this.firestoreStatefulComponentFactory = firestoreStatefulComponentFactory;
      this.rpcQosOptions = rpcQosOptions;
      this.counterFactory = counterFactory;
      this.rpcAttemptContext = V1FnRpcAttemptContext.BatchWrite;
    }

    @Override
    public Context getRpcAttemptContext() {
      return rpcAttemptContext;
    }

    @Override
    public final void populateDisplayData(DisplayData.Builder builder) {
      builder.include("rpcQosOptions", rpcQosOptions);
    }

    @Override
    public void setup() {
      rpcQos = firestoreStatefulComponentFactory.getRpcQos(rpcQosOptions);
      writes = new PriorityQueue<>(WriteElement.COMPARATOR);

      String namespace = rpcAttemptContext.getNamespace();
      writesSuccessful = counterFactory.get(namespace, "writes_successful");
      writesFailedRetryable = counterFactory.get(namespace, "writes_failed_retryable");
      writesFailedNonRetryable = counterFactory.get(namespace, "writes_failed_non-retryable");
    }

    @Override
    public final void startBundle(StartBundleContext c) {
      String project = c.getPipelineOptions().as(GcpOptions.class).getProject();
      String databaseId = c.getPipelineOptions().as(FirestoreOptions.class).getFirestoreDb();
      databaseRootName =
          DatabaseRootName.of(
              requireNonNull(project, "project must be defined on GcpOptions of PipelineOptions"),
              requireNonNull(
                  databaseId,
                  "firestoreDb must be defined on FirestoreOptions of PipelineOptions"));
      firestoreStub = firestoreStatefulComponentFactory.getFirestoreStub(c.getPipelineOptions());
    }

    /**
     * For each element extract and enqueue all writes from the commit. Then potentially flush any
     * previously and currently enqueued writes.
     *
     *
In order for writes to be enqueued the value of {@link BatchWriteRequest#getDatabase()}
     * must match exactly with the database name this instance is configured for via the provided
     * {@link org.apache.beam.sdk.options.PipelineOptions PipelineOptions}
     *
     *
{@inheritDoc}
     */
    @Override
    public void processElement(ProcessContext context, BoundedWindow window) throws Exception {
      @SuppressWarnings(
          "nullness") // error checker is configured to treat any method not explicitly annotated as
      // @Nullable as non-null, this includes Objects.requireNonNull
      Write write = requireNonNull(context.element(), "context.element() must be non null");
      ProcessContextAdapter<OutT> contextAdapter = new ProcessContextAdapter<>(context);
      int serializedSize = write.getSerializedSize();
      boolean tooLarge = rpcQos.bytesOverLimit(serializedSize);
      if (tooLarge) {
        String message =
            String.format(
                "%s for document '%s' larger than configured max allowed bytes per batch",
                getWriteType(write), getName(write));
        handleWriteFailures(
            contextAdapter,
            clock.instant(),
            ImmutableList.of(
                KV.of(
                    new WriteFailure(
                        write,
                        WriteResult.newBuilder().build(),
                        Status.newBuilder()
                            .setCode(Code.INVALID_ARGUMENT.getNumber())
                            .setMessage(message)
                            .build()),
                    window)),
            () -> LOG.info(message));
      } else {
        writes.offer(new WriteElement(queueNextEntryPriority++, write, window));
        flushBatch(/* finishingBundle */ false, contextAdapter);
      }
    }

    /**
     * Attempt to flush any outstanding enqueued writes before cleaning up any bundle related state.
     * {@inheritDoc}
     */
    @SuppressWarnings("nullness") // allow clearing transient fields
    @Override
    public void finishBundle(FinishBundleContext context) throws Exception {
      try {
        flushBatch(/* finishingBundle */ true, new FinishBundleContextAdapter<>(context));
      } finally {
        databaseRootName = null;
        firestoreStub.close();
      }
    }

    /**
     * Possibly flush enqueued writes to Firestore.
     *
     *
This flush attempts to maximize throughput and success rate of RPCs. When a flush should
     * happen and how many writes are included is determined and managed by the {@link RpcQos}
     * instance of this class.
     *
     * @param finishingBundle A boolean specifying if this call is from {@link
     *     #finishBundle(DoFn.FinishBundleContext)}. If {@code true}, this method will not return
     *     until a terminal state (success, attempts exhausted) for all enqueued writes is reached.
     *     If {@code false} and the batch buffer is not full, this method will yield until another
     *     element is added.
     * @throws InterruptedException If the current thread is interrupted at anytime, such as while
     *     waiting for the next attempt
     * @see RpcQos
     * @see RpcQos.RpcWriteAttempt
     * @see BackOffUtils#next(org.apache.beam.sdk.util.Sleeper, org.apache.beam.sdk.util.BackOff)
     */
    private void flushBatch(boolean finishingBundle, ContextAdapter<OutT> context)
        throws InterruptedException {
      while (!writes.isEmpty()) {
        RpcWriteAttempt attempt = rpcQos.newWriteAttempt(getRpcAttemptContext());
        Instant begin = clock.instant();
        if (!attempt.awaitSafeToProceed(begin)) {
          continue;
        }

        FlushBuffer<WriteElement> flushBuffer = getFlushBuffer(attempt, begin);
        if (flushBuffer.isFull() || (finishingBundle && flushBuffer.isNonEmpty())) {
          DoFlushStatus flushStatus = doFlush(attempt, flushBuffer, context);
          if (flushStatus == DoFlushStatus.ONE_OR_MORE_FAILURES && !finishingBundle) {
            break;
          }
        } else {
          // since we're not going to perform a flush, we need to return the writes that were
          // preemptively removed
          flushBuffer.forEach(writes::offer);
          if (!finishingBundle) {
            // we're not on the final flush, so yield until more elements are delivered or
            // finishingBundle is true
            return;
          }
        }
      }
      if (writes.isEmpty()) {
        // now that the queue has been emptied reset our priority back to 0 to try and ensure
        // we won't run into overflow issues if a worker runs for a long time and processes
        // many writes.
        queueNextEntryPriority = 0;
      }
    }

    private FlushBuffer<WriteElement> getFlushBuffer(RpcWriteAttempt attempt, Instant start) {
      FlushBuffer<WriteElement> buffer = attempt.newFlushBuffer(start);

      WriteElement peek;
      while ((peek = writes.peek()) != null) {
        if (buffer.offer(peek)) {
          writes.poll();
        } else {
          break;
        }
      }
      return buffer;
    }

    private BatchWriteRequest getBatchWriteRequest(FlushBuffer<WriteElement> flushBuffer) {
      BatchWriteRequest.Builder commitBuilder =
          BatchWriteRequest.newBuilder().setDatabase(databaseRootName.toString());
      for (WriteElement element : flushBuffer) {
        commitBuilder.addWrites(element.getValue());
      }
      return commitBuilder.build();
    }

    private DoFlushStatus doFlush(
        RpcWriteAttempt attempt,
        FlushBuffer<WriteElement> flushBuffer,
        ContextAdapter<OutT> context)
        throws InterruptedException {
      int writesCount = flushBuffer.getBufferedElementsCount();
      long bytes = flushBuffer.getBufferedElementsBytes();
      BatchWriteRequest request = getBatchWriteRequest(flushBuffer);
      // This is our retry loop
      //    If an error is encountered and is retryable, continue will start the loop over again
      //    If an error is encountered and is not retryable, the error will be thrown and the loop
      // will end
      //    If no error is encountered the responses WriteResults will be inspected before breaking
      // the loop
      while (true) {
        Instant start = clock.instant();
        LOG.debug(
            "Sending BatchWrite request with {} writes totalling {} bytes", writesCount, bytes);
        Instant end;
        BatchWriteResponse response;
        try {
          attempt.recordRequestStart(start, writesCount);
          response = firestoreStub.batchWriteCallable().call(request);
          end = clock.instant();
          attempt.recordRequestSuccessful(end);
        } catch (RuntimeException exception) {
          end = clock.instant();
          String exceptionMessage = exception.getMessage();
          LOG.warn(
              "Sending BatchWrite request with {} writes totalling {} bytes failed due to error: {}",
              writesCount,
              bytes,
              exceptionMessage != null ? exceptionMessage : exception.getClass().getName());
          attempt.recordRequestFailed(end);
          attempt.recordWriteCounts(end, 0, writesCount);
          flushBuffer.forEach(writes::offer);
          attempt.checkCanRetry(end, exception);
          continue;
        }

        long elapsedMillis = end.minus(Duration.millis(start.getMillis())).getMillis();

        int okCount = 0;
        long okBytes = 0L;
        BoundedWindow okWindow = null;
        List<KV<WriteFailure, BoundedWindow>> nonRetryableWrites = new ArrayList<>();

        List<WriteResult> writeResultList = response.getWriteResultsList();
        List<Status> statusList = response.getStatusList();

        Iterator<WriteElement> iterator = flushBuffer.iterator();
        for (int i = 0; iterator.hasNext() && i < statusList.size(); i++) {
          WriteElement writeElement = iterator.next();
          Status writeStatus = statusList.get(i);
          Code code = Code.forNumber(writeStatus.getCode());

          if (code == Code.OK) {
            okCount++;
            okBytes += writeElement.getSerializedSize();
            okWindow = writeElement.window;
          } else {
            if (attempt.isCodeRetryable(code)) {
              writes.offer(writeElement);
            } else {
              nonRetryableWrites.add(
                  KV.of(
                      new WriteFailure(
                          writeElement.getValue(), writeResultList.get(i), writeStatus),
                      writeElement.window));
            }
          }
        }

        int nonRetryableCount = nonRetryableWrites.size();
        int retryableCount = writesCount - okCount - nonRetryableCount;

        writesSuccessful.inc(okCount);
        writesFailedRetryable.inc(retryableCount);
        writesFailedNonRetryable.inc(nonRetryableCount);

        attempt.recordWriteCounts(end, okCount, nonRetryableCount + retryableCount);
        if (okCount == writesCount) {
          handleWriteSummary(
              context,
              end,
              KV.of(new WriteSuccessSummary(okCount, okBytes), coerceNonNull(okWindow)),
              () ->
                  LOG.debug(
                      "Sending BatchWrite request with {} writes totalling {} bytes was completely applied in {}ms",
                      writesCount,
                      bytes,
                      elapsedMillis));
          attempt.completeSuccess();
          return DoFlushStatus.OK;
        } else {
          if (nonRetryableCount > 0) {
            int finalOkCount = okCount;
            handleWriteFailures(
                context,
                end,
                ImmutableList.copyOf(nonRetryableWrites),
                () ->
                    LOG.warn(
                        "Sending BatchWrite request with {} writes totalling {} bytes was incompletely applied in {}ms ({} ok, {} retryable, {} non-retryable)",
                        writesCount,
                        bytes,
                        elapsedMillis,
                        finalOkCount,
                        retryableCount,
                        nonRetryableCount));
          } else if (retryableCount > 0) {
            int finalOkCount = okCount;
            Runnable logMessage =
                () ->
                    LOG.debug(
                        "Sending BatchWrite request with {} writes totalling {} bytes was incompletely applied in {}ms ({} ok, {} retryable)",
                        writesCount,
                        bytes,
                        elapsedMillis,
                        finalOkCount,
                        retryableCount);
            if (okCount > 0) {
              handleWriteSummary(
                  context,
                  end,
                  KV.of(new WriteSuccessSummary(okCount, okBytes), coerceNonNull(okWindow)),
                  logMessage);
            } else {
              logMessage.run();
            }
          }
          return DoFlushStatus.ONE_OR_MORE_FAILURES;
        }
      }
    }

    /**
     * Window values are part of the WriteElement which is used to full the flushBuffer and
     * accessible after the response is returned. Our FlushBuffer is ensured to be non-empty before
     * passed to this method, so the loop above will always iterate at least once and by virtue of
     * this method only being when at least one ok status OK is present successfulWindow will be
     * non-null.
     *
     *
This method is here to prove to error prone that the value of successfulWindow we are
     * passing along is in fact non-null.
     */
    private static BoundedWindow coerceNonNull(@Nullable BoundedWindow successfulWindow) {
      if (successfulWindow == null) {
        throw new IllegalStateException("Unable to locate window for successful request");
      }
      return successfulWindow;
    }

    private enum DoFlushStatus {
      OK,
      ONE_OR_MORE_FAILURES
    }

    abstract void handleWriteFailures(
        ContextAdapter<OutT> context,
        Instant timestamp,
        List<KV<WriteFailure, BoundedWindow>> writeFailures,
        Runnable logMessage);

    abstract void handleWriteSummary(
        ContextAdapter<OutT> context,
        Instant timestamp,
        KV<WriteSuccessSummary, BoundedWindow> tuple,
        Runnable logMessage);

    private static String getWriteType(Write w) {
      if (w.hasUpdate()) {
        return "UPDATE";
      } else if (w.hasTransform()) {
        return "TRANSFORM";
      } else {
        return "DELETE";
      }
    }

    private static String getName(Write w) {
      if (w.hasUpdate()) {
        return w.getUpdate().getName();
      } else if (w.hasTransform()) {
        return w.getTransform().getDocument();
      } else {
        return w.getDelete();
      }
    }

    /**
     * Adapter interface which provides a common parent for {@link ProcessContext} and {@link
     * FinishBundleContext} so that we are able to use a single common invocation to output from.
     */
    interface ContextAdapter<T> {
      void output(T t, Instant timestamp, BoundedWindow window);
    }

    private static final class ProcessContextAdapter<T> implements ContextAdapter<T> {
      private final DoFn<Write, T>.ProcessContext context;

      private ProcessContextAdapter(DoFn<Write, T>.ProcessContext context) {
        this.context = context;
      }

      @Override
      public void output(T t, Instant timestamp, BoundedWindow window) {
        context.outputWithTimestamp(t, timestamp);
      }
    }

    private static final class FinishBundleContextAdapter<T> implements ContextAdapter<T> {
      private final DoFn<Write, T>.FinishBundleContext context;

      private FinishBundleContextAdapter(DoFn<Write, T>.FinishBundleContext context) {
        this.context = context;
      }

      @Override
      public void output(T t, Instant timestamp, BoundedWindow window) {
        context.output(t, timestamp, window);
      }
    }
  }

  static final class WriteElement implements Element<Write> {

    private static final Comparator<WriteElement> COMPARATOR =
        Comparator.comparing(WriteElement::getQueuePosition);
    private final int queuePosition;
    private final Write value;

    private final BoundedWindow window;

    WriteElement(int queuePosition, Write value, BoundedWindow window) {
      this.value = value;
      this.queuePosition = queuePosition;
      this.window = window;
    }

    public int getQueuePosition() {
      return queuePosition;
    }

    @Override
    public Write getValue() {
      return value;
    }

    @Override
    public long getSerializedSize() {
      return value.getSerializedSize();
    }

    @Override
    public boolean equals(@Nullable Object o) {
      if (this == o) {
        return true;
      }
      if (!(o instanceof WriteElement)) {
        return false;
      }
      WriteElement that = (WriteElement) o;
      return queuePosition == that.queuePosition
          && value.equals(that.value)
          && window.equals(that.window);
    }

    @Override
    public int hashCode() {
      return Objects.hash(queuePosition, value, window);
    }

    @Override
    public String toString() {
      return "WriteElement{"
          + "queuePosition="
          + queuePosition
          + ", value="
          + value
          + ", window="
          + window
          + '}';
    }
  }
}


text:
/**
 * Youtube room chat-plugin.
 * Supports adding channels and selecting a random channel.
 * Also supports showing video data on request.
 * Written by Mia, with some design concepts from bumbadadabum.
 * @author mia-pi-git
 */

import {Utils, FS, Net} from '../../lib';

const ROOT = 'https://www.googleapis.com/youtube/v3/';
const STORAGE_PATH = 'config/chat-plugins/youtube.json';

export const videoDataCache: Map<string, VideoData> = Chat.oldPlugins.youtube?.videoDataCache || new Map();
export const searchDataCache: Map<string, string[]> = Chat.oldPlugins.youtube?.searchDataCache || new Map();

interface ChannelEntry {
	name: string;
	description: string;
	url: string;
	icon: string;
	videos: number;
	subs: number;
	views: number;
	username?: string;
	category?: string;
}

export interface VideoData {
	id: string;
	title: string;
	date: string;
	description: string;
	channelTitle: string;
	channelUrl: string;
	views: number;
	thumbnail: string;
	likes: number;
	dislikes: number;
}

interface TwitchChannel {
	status: string;
	display_name: string;
	name: string;
	language: string;
	created_at: string;
	logo: string;
	views: number;
	followers: number;
	video_banner: string;
	url: string;
	game: string;
	description: string;
	updated_at: string;
}

interface ChannelData {
	channels: {[k: string]: ChannelEntry};
	categories: string[];
	intervalTime?: number;
}

function loadData() {
	const raw: AnyObject = JSON.parse(FS(STORAGE_PATH).readIfExistsSync() || "{}");
	if (!(raw.channels && raw.categories)) { // hasn't been converted to new format
		const data: Partial<ChannelData> = {};
		data.channels = raw;
		data.categories = [];
		// re-save into new format
		FS(STORAGE_PATH).writeUpdate(() => JSON.stringify(data));
		return data as ChannelData;
	}
	return raw as ChannelData;
}

const channelData: ChannelData = loadData();

export class YoutubeInterface {
	interval: NodeJS.Timer | null;
	intervalTime: number;
	data: ChannelData;
	linkRegex = /^(https?:\/\/)?(www\.)?(youtube\.com|youtu\.be)(\/|$)/i;
	constructor(data?: ChannelData) {
		this.data = data ? data : {categories: [], channels: {}};
		this.interval = null;
		this.intervalTime = 0;
		if (data?.intervalTime) {
			this.runInterval(`${data.intervalTime}`);
		}
	}
	async getChannelData(link: string, username?: string) {
		if (!Config.youtubeKey) {
			throw new Chat.ErrorMessage(`This server does not support YouTube commands. If you're the owner, you can enable them by setting up Config.youtubekey.`);
		}
		const id = this.getId(link);
		const raw = await Net(`${ROOT}channels`).get({
			query: {part: 'snippet,statistics', id, key: Config.youtubeKey},
		});
		const res = JSON.parse(raw);
		if (!res?.items || res.items.length < 1) {
			throw new Chat.ErrorMessage(`Channel not found.`);
		}
		const data = res.items[0];
		const cache: ChannelEntry = {
			name: data.snippet.title,
			description: data.snippet.description,
			url: data.snippet.customUrl,
			icon: data.snippet.thumbnails.medium.url,
			videos: Number(data.statistics.videoCount),
			subs: Number(data.statistics.subscriberCount),
			views: Number(data.statistics.viewCount),
			username: username,
		};
		this.data.channels[id] = {...cache};
		this.save();
		return cache;
	}
	async generateChannelDisplay(link: string) {
		const id = this.getId(link);
		const {name, description, icon, videos, subs, views, username} = await this.get(id);
		// credits bumbadadabum for most of the html
		let buf = `<div class="infobox"><table style="margin:0px;"><tr>`;
		buf += `<td style="margin:5px;padding:5px;min-width:175px;max-width:160px;text-align:center;border-bottom:0px;">`;
		buf += `<div style="padding:5px;background:white;border:1px solid black;margin:auto;max-width:100px;max-height:100px;">`;
		buf += `<a href="${ROOT}channel/${id}"><img src="${icon}" width=100px height=100px/></a>`;
		buf += `</div><p style="margin:5px 0px 4px 0px;word-wrap:break-word;">`;
		buf += `<a style="font-weight:bold;color:#c70000;font-size:12pt;" href="https://www.youtube.com/channel/${id}">${name}</a>`;
		buf += `</p></td><td style="padding: 0px 25px;font-size:10pt;background:rgb(220,20,60);width:100%;border-bottom:0px;vertical-align:top;">`;
		buf += `<p style="padding: 5px;border-radius:8px;color:white;font-weight:bold;text-align:center;">`;
		buf += `${videos} videos | ${subs} subscribers | ${views} video views</p>`;
		buf += `<p style="margin-left: 5px; font-size:9pt;color:white;">`;
		buf += `${description.slice(0, 400).replace(/\n/g, ' ')}${description.length > 400 ? '(...)' : ''}</p>`;
		if (username) {
			buf += `<p style="text-align:left;font-style:italic;color:white;">PS username: ${username}</p></td></tr></table></div>`;
		} else {
			buf += '

';
		}
		return buf;
	}
	randChannel(cat?: string) {
		let channels = Object.keys(this.data.channels);
		if (channels.length < 1) {
			throw new Chat.ErrorMessage(`There are no channels in the database.`);
		}
		if (cat) {
			cat = toID(cat);
			const categoryIDs = this.data.categories.map(toID);
			if (!categoryIDs.includes(cat as ID)) {
				throw new Chat.ErrorMessage(`Invalid category.`);
			}
			channels = channels.filter(id => {
				const channel = this.data.channels[id];
				return channel.category && toID(channel.category) === cat;
			});
		}

		const id = Utils.shuffle(channels)[0];
		return this.generateChannelDisplay(id);
	}
	get(id: string, username?: string): Promise<ChannelEntry> {
		if (!(id in this.data.channels)) return this.getChannelData(id, username);
		return Promise.resolve({...this.data.channels[id]});
	}
	async getVideoData(id: string): Promise<VideoData | null> {
		const cached = videoDataCache.get(id);
		if (cached) return cached;
		let raw;
		try {
			raw = await Net(`${ROOT}videos`).get({
				query: {part: 'snippet,statistics', id, key: Config.youtubeKey},
			});
		} catch (e: any) {
			throw new Chat.ErrorMessage(`Failed to retrieve video data: ${e.message}.`);
		}
		const res = JSON.parse(raw);
		if (!res?.items || res.items.length < 1) return null;
		const video = res.items[0];
		const data: VideoData = {
			title: video.snippet.title,
			id,
			date: new Date(video.snippet.publishedAt).toString(),
			description: video.snippet.description,
			channelTitle: video.snippet.channelTitle,
			channelUrl: video.snippet.channelId,
			views: video.statistics.viewCount,
			thumbnail: video.snippet.thumbnails.default.url,
			likes: video.statistics.likeCount,
			dislikes: video.statistics.dislikeCount,
		};
		videoDataCache.set(id, data);
		return data;
	}
	channelSearch(search: string) {
		let channel;
		if (this.data.channels[search]) {
			channel = search;
		} else {
			for (const id of Object.keys(this.data.channels)) {
				const name = toID(this.data.channels[id].name);
				const username = this.data.channels[id].username;
				if (name === toID(search) || username && toID(username) === toID(search)) {
					channel = id;
					break; // don't iterate through everything once a match is found
				}
			}
		}
		return channel;
	}
	getId(link: string) {
		let id = '';
		if (!link) throw new Chat.ErrorMessage('You must provide a YouTube link.');
		if (this.data.channels[link]) return link;
		if (!link.includes('channel/')) {
			if (link.includes('youtube')) {
				id = link.split('v=')[1] || '';
			} else if (link.includes('youtu.be')) {
				id = link.split('/')[3] || '';
			} else {
				throw new Chat.ErrorMessage('Invalid YouTube channel link.');
			}
		} else {
			id = link.split('channel/')[1] || '';
		}
		if (id.includes('&')) id = id.split('&')[0];
		if (id.includes('?')) id = id.split('?')[0];
		return id;
	}
	async generateVideoDisplay(link: string, fullInfo = false) {
		if (!Config.youtubeKey) {
			throw new Chat.ErrorMessage(`This server does not support YouTube commands. If you're the owner, you can enable them by setting up Config.youtubekey.`);
		}
		const id = this.getId(link);
		const info = await this.getVideoData(id);
		if (!info) throw new Chat.ErrorMessage(`Video not found.`);
		if (!fullInfo) {
			let buf = `<b>${info.title}</b> `;
			buf += `(<a class="subtle" href="https://youtube.com/channel/${info.channelUrl}">${info.channelTitle}</a>)<br />`;
			buf += `<youtube src="https://www.youtube.com/embed/${id}" />`;
			return buf;
		}
		let buf = `<table style="margin:0px;"><tr>`;
		buf += `<td style="margin:5px;padding:5px;min-width:175px;max-width:160px;text-align:center;border-bottom:0px;">`;
		buf += `<div style="padding:5px;background:#b0b0b0;border:1px solid black;margin:auto;max-width:100px;max-height:100px;">`;
		buf += `<a href="${ROOT}channel/${id}"><img src="${info.thumbnail}" width=100px height=100px/></a>`;
		buf += `</div><p style="margin:5px 0px 4px 0px;word-wrap:break-word;">`;
		buf += `<a style="font-weight:bold;color:#c70000;font-size:12pt;" href="https://www.youtube.com/watch?v=${id}">${info.title}</a>`;
		buf += `</p></td><td style="padding: 0px 25px;font-size:10pt;max-width:100px;background:`;
		buf += `#white;width:100%;border-bottom:0px;vertical-align:top;">`;
		buf += `<p style="background: #e22828; padding: 5px;border-radius:8px;color:white;font-weight:bold;text-align:center;">`;
		buf += `${info.likes} likes | ${info.dislikes} dislikes | ${info.views} video views<br><br>`;
		buf += `<small>Published on ${info.date} | ID: ${id}</small><br>Uploaded by: ${info.channelTitle}</p>`;
		buf += `<br><details><summary>Video Description</p></summary>`;
		buf += `<p style="background: #e22828;max-width:500px;padding: 5px;border-radius:8px;color:white;font-weight:bold;text-align:center;">`;
		buf += `<i>${info.description.slice(0, 400).replace(/\n/g, ' ')}${info.description.length > 400 ? '(...)' : ''}</p><i></details></td>`;
		return buf;
	}
	save() {
		return FS(STORAGE_PATH).writeUpdate(() => JSON.stringify(this.data));
	}
	async searchVideo(name: string, limit?: number): Promise<string[] | undefined> {
		const cached = searchDataCache.get(toID(name));
		if (cached) {
			return cached.slice(0, limit);
		}
		const raw = await Net(`${ROOT}search`).get({
			query: {
				part: 'snippet', q: name,
				key: Config.youtubeKey, order: 'relevance',
			},
		});
		const result = JSON.parse(raw);
		const resultArray = result.items?.map((item: AnyObject) => item?.id?.videoId).filter(Boolean);
		searchDataCache.set(toID(name), resultArray);
		return resultArray.slice(0, limit);
	}
	async searchChannel(name: string, limit = 10): Promise<string[] | undefined> {
		const raw = await Net(`${ROOT}search`).get({
			query: {
				part: 'snippet', q: name, type: 'channel',
				key: Config.youtubeKey, order: 'relevance', maxResults: limit,
			},
		});
		const result = JSON.parse(raw);
		return result?.items.map((item: AnyObject) => item?.snippet?.channelId);
	}
	runInterval(time: string) {
		let interval = Number(time);
		if (interval < 10) throw new Chat.ErrorMessage(`${interval} is too low - set it above 10 minutes.`);
		this.intervalTime = interval;
		this.data.intervalTime = interval;
		interval = interval * 60 * 1000;
		if (this.interval) clearInterval(this.interval);
		this.interval = setInterval(() => {
			void (async () => {
				const room = Rooms.get('youtube');
				if (!room) return; // do nothing if the room doesn't exist anymore
				const res = await YouTube.randChannel();
				room.add(`|html|${res}`).update();
			})();
		}, interval);
		return this.interval;
	}
	async createGroupWatch(url: string, baseRoom: Room, title: string) {
		const id = this.getId(url);
		const videoInfo = await this.getVideoData(id);
		if (!videoInfo) throw new Chat.ErrorMessage(`Video not found.`);
		if ([...Rooms.rooms.values()].some(r => r.roomid.startsWith('video-watch-'))) {
			throw new Chat.ErrorMessage(
				`A groupwatch is already going on. Please wait until it is done before creating another.`
			);
		}
		const num = baseRoom.nextGameNumber();
		baseRoom.saveSettings();
		const gameRoom = Rooms.createGameRoom(`video-watch-${num}` as RoomID, Utils.html`[Group Watch] ${title}`, {
			isPrivate: 'hidden',
		});
		const game = new GroupWatch(gameRoom, url, videoInfo);
		gameRoom.game = game;
		gameRoom.setParent(baseRoom);
		return gameRoom;
	}
}

export const Twitch = new class {
	linkRegex = /(https?:\/\/)?twitch.tv\/([A-Za-z0-9]+)/i;
	async getChannel(channel: string): Promise<TwitchChannel | undefined> {
		channel = toID(channel);
		let res;
		try {
			res = await Net(`https://api.twitch.tv/kraken/search/channels`).get({
				headers: {
					'Client-Id': Config.twitchKey,
					'Content-Type': 'application/json',
					'Accept': "application/vnd.twitchtv.v5+json",
				},
				query: {query: channel},
			});
		} catch (e: any) {
			throw new Chat.ErrorMessage(`Error retrieving twitch channel: ${e.message}`);
		}
		const data = JSON.parse(res);
		Utils.sortBy(data.channels as AnyObject[], c => -c.followers);
		return data?.channels?.[0] as TwitchChannel | undefined;
	}
	visualizeChannel(info: TwitchChannel) {
		let buf = `<div class="infobox"><table style="margin:0px;"><tr>`;
		buf += `<td style="margin:5px;padding:5px;min-width:175px;max-width:160px;text-align:center;border-bottom:0px;">`;
		buf += `<div style="padding:5px;background:white;border:1px solid black;margin:auto;max-width:100px;max-height:100px;">`;
		buf += `<a href="${info.url}"><img src="${info.logo}" width=100px height=100px/></a>`;
		buf += `</div><p style="margin:5px 0px 4px 0px;word-wrap:break-word;">`;
		buf += `<a style="font-weight:bold;color:#6441a5;font-size:12pt;" href="${info.logo}">${info.display_name}</a>`;
		buf += `</p></td><td style="padding: 0px 25px;font-size:10pt;background:rgb(100, 65, 164);width:100%;border-bottom:0px;vertical-align:top;">`;
		buf += `<p style="padding: 5px;border-radius:8px;color:white;font-size:15px;font-weight:bold;text-align:center;">`;
		const created = new Date(info.created_at);
		buf += `${info.followers} subscribers | ${info.views} stream views | created ${Chat.toTimestamp(created).split(' ')[0]}</p>`;
		buf += `<p style="color:white;font-size:10px">Last seen playing ${info.game} (Status: ${info.status})</p>`;
		buf += `<hr /><p style="margin-left: 5px; font-size:9pt;color:white;">`;
		buf += `${info.description.slice(0, 400).replace(/\n/g, ' ')}${info.description.length > 400 ? '...' : ''}</p>`;
		buf += '

';
		return buf;
	}
};

export class GroupWatch extends Rooms.SimpleRoomGame {
	url: string;
	info: VideoData;
	started: number | null = null;
	constructor(room: Room, url: string, videoInfo: VideoData) {
		super(room);
		this.url = url;
		this.info = videoInfo;
		this.controls(`<h2><i>Waiting to start the video...</i></h2>`);
	}
	onJoin(user: User) {
		const hints = this.hints();
		for (const hint of hints) {
			user.sendTo(this.room.roomid, `|html|${hint}`);
		}
	}
	start() {
		if (this.started) throw new Chat.ErrorMessage(`We've already started.`);
		this.controls(this.getStatsDisplay());
		this.field(this.getVideoDisplay());
		this.started = Date.now();
		this.add(`|html|<h2>Group Watch!</h2>`);
	}
	hints() {
		const hints = [
			`To watch, all you need to do is click play on the video once staff have started it!`,
			`We are currently watching: <a href="${this.url}">${this.info.title}</a>`,
		];
		if (this.started) {
			const diff = Date.now() - this.started;
			hints.push(`Video is currently at ${Chat.toDurationString(diff)} (${diff / 1000} seconds)`);
		}
		return hints;
	}
	getStatsDisplay() {
		let controlsHTML = `<h3>${this.info.title}</h3>`;
		controlsHTML += `<div class="infobox"><b>Channel:</b> `;
		controlsHTML += `<a href="https://www.youtube.com/channel/${this.info.channelUrl}">${this.info.channelTitle}</a><br />`;
		controlsHTML += `<b>Likes:</b> ${this.info.likes} | <b>Dislikes:</b> ${this.info.dislikes}<br />`;
		controlsHTML += `<b>Uploaded:</b> ${Chat.toTimestamp(new Date(this.info.date))}<br />`;
		controlsHTML += `<details><summary>Description</summary>${this.info.description.replace(/\n/ig, '<br />')}</details>`;
		controlsHTML += `</div>`;
		return controlsHTML;
	}
	getVideoDisplay() {
		let buf = `<p style="background: #e22828; padding: 5px;border-radius:8px;color:white;font-weight:bold;text-align:center;">`;
		buf += `<br /><br /><b>${this.info.title}</b><br />`;
		const id = YouTube.getId(this.url);
		const url = `https://youtube.com/watch?v=${id}`;
		buf += `<youtube src="${url}" />`;
		buf += `<br />`.repeat(4);
		buf += `</p>`;
		return buf;
	}
	controls(html: string) {
		this.add(`|controlshtml|<center>${html}</center>`);
	}
	field(html: string) {
		this.add(`|fieldhtml|${html}`);
	}
	add(buf: string) {
		this.room.add(buf).update();
	}
	destroy() {
		this.controls(`<b>The group watch has ended.</b>`);
		let endBuf = `<center>`;
		endBuf += this.getStatsDisplay();
		endBuf += `<br /> Thanks for watching!</center>`;
		this.field(endBuf);
		this.room = null!;
	}
}

export class TwitchStream extends Rooms.SimpleRoomGame {
	started = false;
	data: TwitchChannel;
	constructor(room: Room, data: TwitchChannel) {
		super(room);
		this.data = data;
	}
	static async createStreamWatch(room: Room, channel: string) {
		if ([...Rooms.rooms.values()].some(
			r => r.roomid.startsWith(`twitch-`) && r.parent?.roomid === room?.roomid
		)) {
			throw new Chat.ErrorMessage(`Twitch watch already in progress for this room.`);
		}
		const data = await Twitch.getChannel(channel);
		if (!data) throw new Chat.ErrorMessage(`Channel not found`);
		const watchRoom = Rooms.createGameRoom(
			`twitch-stream-watch-${room.nextGameNumber()}` as RoomID,
			Utils.html`[Twitch Watch] ${data.display_name}`,
			{isPrivate: 'hidden'}
		);
		room.saveSettings();
		watchRoom.setParent(room);
		const stream = new TwitchStream(watchRoom, data);
		watchRoom.game = stream;
		return watchRoom;
	}
	field(buf: string) {
		this.add(`|fieldhtml|${buf}`);
	}
	controls(buf: string) {
		this.add(`|controlshtml|${buf}`);
	}
	add(data: string) {
		return this.room.add(data).update();
	}
	onJoin(user: User) {
		if (!user.named) return;
		this.controls(this.getControlsDisplay());
	}
	onLeave(user: User) {
		if (!user.named) return;
		this.controls(this.getControlsDisplay());
	}
	start() {
		if (this.started) {
			throw new Chat.ErrorMessage(`Stream already started`);
		}
		this.controls(this.getControlsDisplay());
		this.field(this.getStreamDisplay());
		this.add(`|html|<h2>The stream watch has started!</h2>`);
		this.started = true;
	}
	end() {
		this.field('');
		this.controls(`<center><h2>Stream watch ended</h2></center>`);
		this.room.parent?.add(`|uhtmlchange|ts-${this.room.roomid}|`);
		this.add(`|expire|Stream ended`);
		this.room.destroy();
	}
	getControlsDisplay() {
		let buf = `<p style="background: #6441a5; padding: 5px;border-radius:8px;color:white;font-weight:bold;text-align:center;">`;
		buf += `<strong>Watching <a href="${this.data.url}" class="subtle">${this.data.display_name}</strong><br />`;
		buf += `${Chat.count(Object.keys(this.room.users).length, 'users')} watching<br />`;
		buf += `<strong>Playing: ${this.data.game}`;
		return buf;
	}
	getStreamDisplay() {
		let buf = `<p style="background: #6441a5; padding: 5px;border-radius:8px;color:white;font-weight:bold;text-align:center;">`;
		buf += `<twitch src="${this.data.url}" width="600" height="330" />`;
		return buf;
	}
}

export const YouTube = new YoutubeInterface(channelData);

export function destroy() {
	if (YouTube.interval) clearInterval(YouTube.interval);
}

export const commands: Chat.ChatCommands = {
	async randchannel(target, room, user) {
		room = this.requireRoom('youtube' as RoomID);
		if (Object.keys(YouTube.data.channels).length < 1) return this.errorReply(`No channels in the database.`);
		target = toID(target);
		this.runBroadcast();
		const data = await YouTube.randChannel(target);
		return this.sendReply(`|html|${data}`);
	},
	randchannelhelp: [`/randchannel - View data of a random channel from the YouTube database.`],

	yt: 'youtube',
	youtube: {
		async addchannel(target, room, user) {
			room = this.requireRoom('youtube' as RoomID);
			this.checkCan('mute', null, room);
			const [id, name] = target.split(',').map(t => t.trim());
			if (!id) return this.errorReply('Specify a channel ID.');
			await YouTube.getChannelData(id, name);
			this.modlog('ADDCHANNEL', null, `${id} ${name ? `username: ${name}` : ''}`);
			return this.privateModAction(
				`${user.name} added channel with id ${id} ${name ? `and username (${name}) ` : ''} to the random channel pool.`
			);
		},
		addchannelhelp: [`/addchannel - Add channel data to the YouTube database. Requires: % @ #`],

		removechannel(target, room, user) {
			room = this.requireRoom('youtube' as RoomID);
			this.checkCan('mute', null, room);
			const id = YouTube.channelSearch(target);
			if (!id) return this.errorReply(`Channel with ID or name ${target} not found.`);
			delete YouTube.data.channels[id];
			YouTube.save();
			this.privateModAction(`${user.name} deleted channel with ID or name ${target}.`);
			return this.modlog(`REMOVECHANNEL`, null, id);
		},
		removechannelhelp: [`/youtube removechannel - Delete channel data from the YouTube database. Requires: % @ #`],

		async channel(target, room, user) {
			room = this.requireRoom('youtube' as RoomID);
			const channel = YouTube.channelSearch(target);
			if (!channel) return this.errorReply(`No channels with ID or name ${target} found.`);
			const data = await YouTube.generateChannelDisplay(channel);
			this.runBroadcast();
			return this.sendReply(`|html|${data}`);
		},
		channelhelp: [
			'/youtube channel - View the data of a specified channel. Can be either channel ID or channel name.',
		],
		async video(target, room, user) {
			room = this.requireRoom('youtube' as RoomID);
			this.checkCan('mute', null, room);
			const buffer = await YouTube.generateVideoDisplay(target, true);
			this.runBroadcast();
			this.sendReplyBox(buffer);
		},

		channels(target, room, user) {
			target = toID(target);
			return this.parse(`/j view-channels${target ? `-${target}` : ''}`);
		},
		help(target, room, user) {
			return this.parse('/help youtube');
		},

		categories() {
			return this.parse(`/j view-channels-categories`);
		},

		update(target, room, user) {
			room = this.requireRoom('youtube' as RoomID);
			this.checkCan('mute', null, room);
			const [channel, name] = target.split(',');
			const id = YouTube.channelSearch(channel);
			if (!id) return this.errorReply(`Channel ${channel} is not in the database.`);
			YouTube.data.channels[id].username = name;
			this.modlog(`UPDATECHANNEL`, null, name);
			this.privateModAction(`${user.name} updated channel ${id}'s username to ${name}.`);
			YouTube.save();
		},
		interval: 'repeat',
		repeat(target, room, user) {
			room = this.requireRoom('youtube' as RoomID);
			this.checkCan('declare', null, room);
			if (!target) {
				if (!YouTube.interval) return this.errorReply(`The YouTube plugin is not currently running an interval.`);
				return this.sendReply(`Interval is currently set to ${Chat.toDurationString(YouTube.intervalTime * 60 * 1000)}.`);
			}
			if (this.meansNo(target)) {
				if (!YouTube.interval) return this.errorReply(`The interval is not currently running`);
				clearInterval(YouTube.interval);
				delete YouTube.data.intervalTime;
				YouTube.save();
				this.privateModAction(`${user.name} turned off the YouTube interval`);
				return this.modlog(`YOUTUBE INTERVAL`, null, 'OFF');
			}
			if (Object.keys(channelData).length < 1) return this.errorReply(`No channels in the database.`);
			if (isNaN(parseInt(target))) return this.errorReply(`Specify a number (in minutes) for the interval.`);
			YouTube.runInterval(target);
			YouTube.save();
			this.privateModAction(`${user.name} set a randchannel interval to ${target} minutes`);
			return this.modlog(`CHANNELINTERVAL`, null, `${target} minutes`);
		},
		addcategory(target, room, user) {
			room = this.requireRoom('youtube' as RoomID);
			this.checkCan('mute', null, room);
			const categoryID = toID(target);
			if (!categoryID) return this.parse(`/help youtube`);
			if (YouTube.data.categories.map(toID).includes(categoryID)) {
				return this.errorReply(`This category is already added. To change it, remove it and re-add it.`);
			}
			YouTube.data.categories.push(target);
			this.modlog(`YOUTUBE ADDCATEGORY`, null, target);
			this.privateModAction(`${user.name} added category '${target}' to the categories list.`);
			YouTube.save();
		},
		removecategory(target, room, user) {
			room = this.requireRoom('youtube' as RoomID);
			this.checkCan('mute', null, room);
			const categoryID = toID(target);
			if (!categoryID) return this.parse(`/help youtube`);
			const index = YouTube.data.categories.indexOf(target);
			if (index < 0) {
				return this.errorReply(`${target} is not a valid category.`);
			}
			for (const id in YouTube.data.channels) {
				const channel = YouTube.data.channels[id];
				if (channel.category === target) delete YouTube.data.channels[id].category;
			}
			YouTube.save();
			this.privateModAction(`${user.name} removed the category '${target}' from the category list.`);
			this.modlog(`YOUTUBE REMOVECATEGORY`, null, target);
		},
		setcategory(target, room, user) {
			room = this.requireRoom('youtube' as RoomID);
			this.checkCan('mute', null, room);
			target = target.trim();
			const [category, id] = Utils.splitFirst(target, ',').map(item => item.trim());
			if (!target || !category || !id) {
				return this.parse('/help youtube');
			}
			if (!YouTube.data.categories.includes(category)) {
				return this.errorReply(`Invalid category.`);
			}
			const name = YouTube.channelSearch(id);
			if (!name) return this.errorReply(`Invalid channel.`);
			const channel = YouTube.data.channels[name];
			YouTube.data.channels[name].category = category;
			YouTube.save();
			this.modlog(`YOUTUBE SETCATEGORY`, null, `${id}: to category ${category}`);
			this.privateModAction(`${user.name} set the channel ${channel.name}'s category to '${category}'.`);
		},
		decategorize(target, room, user) {
			room = this.requireRoom('youtube' as RoomID);
			this.checkCan('mute', null, room);
			target = target.trim();
			if (!target) {
				return this.parse('/help youtube');
			}
			const name = YouTube.channelSearch(target);
			if (!name) return this.errorReply(`Invalid channel.`);
			const channel = YouTube.data.channels[name];
			const category = channel.category;
			if (!category) return this.errorReply(`That channel does not have a category.`);
			delete channel.category;
			YouTube.save();
			this.modlog(`YOUTUBE DECATEGORIZE`, null, target);
			this.privateModAction(`${user.name} removed the channel ${channel.name} from the category ${category}.`);
		},
		async groupwatch(target, room, user) {
			room = this.requireRoom('youtube' as RoomID);
			this.checkCan('mute', null, room);
			const [url, title] = Utils.splitFirst(target, ',').map(p => p.trim());
			if (!url || !title) return this.errorReply(`You must specify a video to watch and a title for the group watch.`);
			const gameRoom = await YouTube.createGroupWatch(url, room, title);
			this.modlog(`YOUTUBE GROUPWATCH`, null, `${url} (${title})`);
			room.add(
				`|uhtml|${gameRoom.roomid}|` +
				`<button class="button" name="send" value="/j ${gameRoom.roomid}">Join the ongoing group watch!</button>`
			);
			room.send(`|tempnotify|youtube|New groupwatch - ${title}!`);
			this.update();
			user.joinRoom(gameRoom);
		},
		endwatch(target, room, user) {
			room = this.requireRoom();
			this.checkCan('mute', null, room);
			this.requireGame(GroupWatch);
			room.parent!.modlog({action: `GROUPWATCH END`, loggedBy: user.id});
			room.parent!.add(`|uhtmlchange|${room.roomid}|`).update();
			room.destroy();
		},
		startwatch: 'beginwatch',
		beginwatch(target, room, user) {
			room = this.requireRoom();
			this.checkCan('mute', null, room);
			const game = this.requireGame(GroupWatch);
			game.start();
		},
		groupwatches() {
			let buf = `<strong>Ongoing groupwatches:</strong><br />`;
			for (const curRoom of Rooms.rooms.values()) {
				if (!curRoom.getGame(GroupWatch)) continue;
				buf += `<button class="button" name="send" value="/j ${curRoom.roomid}">${curRoom.title}</button>`;
			}
			this.runBroadcast();
			this.sendReplyBox(buf);
		},
	},
	youtubehelp: [
		`YouTube commands:`,
		`/randchannel [optional category]- View data of a random channel from the YouTube database.` +
			` If a category is given, the random channel will be in the  given category.`,
		`/youtube addchannel [channel] - Add channel data to the YouTube database. Requires: % @ #`,
		`/youtube removechannel [channel]- Delete channel data from the YouTube database. Requires: % @ #`,
		`/youtube channel [channel] - View the data of a specified channel. Can be either channel ID or channel name.`,
		`/youtube video [video] - View data of a specified video. Can be either channel ID or channel name.`,
		`/youtube update [channel], [name] - sets a channel's PS username to [name]. Requires: % @ #`,
		`/youtube repeat [time] - Sets an interval for [time] minutes, showing a random channel each time. Requires: # &`,
		`/youtube addcategory [name] - Adds the [category] to the channel category list. Requires: @ # &`,
		`/youtube removecategory [name] - Removes the [category] from the channel category list. Requires: @ # &`,
		`/youtube setcategory [category], [channel name] - Sets the category for [channel] to [category]. Requires: @ # &`,
		`/youtube decategorize [channel name] - Removes the category for the [channel], if there is one. Requires: @ # &`,
		`/youtube categores - View all channels sorted by category.`,
		`/youtube groupwatch [link], [title] - Creates a group watch of the [url] with the given [title]. Requires % @ & #`,
		`/youtube startwatch - Starts the group watch in the current room, if there is one. Requires % @ & #`,
		`/youtube stopwatch - Ends the current group watch, if there is one in the current room. Requires % @ & #`,
	],
	twitch: {
		async channel(target, room, user) {
			room = this.requireRoom('youtube' as RoomID);
			if (!Config.twitchKey) return this.errorReply(`Twitch is not configured`);
			const data = await Twitch.getChannel(target);
			if (!data) return this.errorReply(`Channel not found`);
			const html = Twitch.visualizeChannel(data);
			this.runBroadcast();
			return this.sendReplyBox(html);
		},
		async watch(target, room, user) {
			room = this.requireRoom();
			if (!['youtube', 'pokemongo'].includes(room.roomid)) {
				throw new Chat.ErrorMessage(`You cannot use this command in this room.`);
			}
			this.checkCan('mute', null, room);
			if (!toID(target)) {
				return this.errorReply(`Invalid channel`);
			}
			const gameRoom = await TwitchStream.createStreamWatch(room, target);
			user.joinRoom(gameRoom);
			room.add(
				`|uhtml|ts-${gameRoom.roomid}|` +
				`<button class="button" name="send" value="/j ${gameRoom.roomid}">Join the ongoing stream watch!</button>`
			).update();
		},
		start(target, room) {
			room = this.requireRoom();
			const stream = this.requireGame(TwitchStream);
			stream.start();
		},
		stop(target, room, user) {
			room = this.requireRoom();
			const stream = this.requireGame(TwitchStream);
			this.checkCan('mute', null, room);
			stream.end();
		},
	},
};

export const pages: Chat.PageTable = {
	async channels(args, user) {
		const [type] = args;
		if (!Config.youtubeKey) return `<h2>Youtube is not configured.</h2>`;
		const titles: {[k: string]: string} = {
			all: 'All channels',
			categories: 'by category',
		};
		const title = titles[type] || 'Usernames only';
		this.title = `[Channels] ${title}`;
		let buffer = `<div class="pad"><h4>Channels in the YouTube database: (${title})`;
		buffer += ` <button class="button" name="send" value="/join view-channels-${type}" style="float: right">Refresh</button>`;
		buffer += `</h4><hr />`;
		switch (toID(type)) {
		case 'categories':
			if (!YouTube.data.categories.length) {
				return this.errorReply(`There are currently no categories in the Youtube channel database.`);
			}
			const sorted: {[k: string]: string[]} = {};
			const channels = YouTube.data.channels;
			for (const [id, channel] of Object.entries(channels)) {
				const category = channel.category || "No category";
				if (!sorted[category]) {
					sorted[category] = [];
				}
				sorted[category].push(id);
			}
			for (const cat in sorted) {
				buffer += `<h3>${cat}:</h3>`;
				for (const id of sorted[cat]) {
					const channel = channels[id];
					buffer += `<details><summary>${channel.name}</summary>`;
					buffer += await YouTube.generateChannelDisplay(id);
					buffer += `</details><br />`;
				}
			}
			break;
		default:
			for (const id of Utils.shuffle(Object.keys(YouTube.data.channels))) {
				const {name, username} = await YouTube.get(id);
				if (toID(type) !== 'all' && !username) continue;
				buffer += `<details><summary>${name}`;
				buffer += `<small><i> (Channel ID: ${id})</i></small>`;
				if (username) buffer += ` <small>(PS name: ${username})</small>`;
				buffer += `</summary>`;
				buffer += await YouTube.generateChannelDisplay(id);
				buffer += `</details><hr/ >`;
			}
			break;
		}
		buffer += `</div>`;
		return buffer;
	},
};


text:
	3p8_kookospahkina_puu
	Uses:		It's a cononut tree.
				Grows on its own up until a certain global number.
					Players can still cultivate it further though.

	History:	Refactored version of the old micro_item_salainen_kookospahkina_puu (all 8 of them...)

	Memes:		puu on kaunis
				cocnut21

	Todo:

	Realizations: self:SetSubMaterial(...,...) DOES NOTHING
				fancy timers and fancy genes are hard to do right... something about local vs. self?
				The invisible block on a log is why there are invis blocks everywhere.
]]

AddCSLuaFile()

ENT.Type = "anim"

ENT.ItemModel = "models/hunter/misc/sphere025x025.mdl"

ENT.lastModel = {"models/props_foliage/mall_palm01_medium.mdl", "models/props_foliage/mall_palm01.mdl"}

function ENT:Use(ply)
	--plant the coconut
	if !self.isPlanted then
		self:GetPhysicsObject():Wake()
		self:Upgrayed()
	end
end

function ENT:Initialize()
	if SERVER then
		--global variable for the softcap of "nature" autoplanting coconuts
		GLOBAL_coconut = GLOBAL_coconut + 1

		self:SetMaterial("models/props_pipes/guttermetal01a")
		self:SetModel(self.ItemModel)
		self:PhysicsInitStandard()
		--self:PhysicsInit(SOLID_VPHYSICS)
		--self:SetSolid(SOLID_VPHYSICS)
		--stop movement
		--self:SetMoveType(MOVETYPE_VPHYSICS)

		local phys = self:GetPhysicsObject()
		if (phys:IsValid()) then
			--phys:Sleep()
			--make float (bouyancy)
			phys:SetBuoyancyRatio(1.0) --0 min, 1 max (wood)
			timer.Simple( 3, function ()
				phys:Sleep()
			end)
		end
		self.isPlanted = false

		self.numberOfFruitMax = 5

		self.treeLevel = 0
		--self.plantTime = math.random(10, 50)
		self.angle1 = math.random(-9,9)
		self.angle2 = math.random(-179,180)

		self.timer = 0
		self.longTimer = 0

		self.health = 75

		self.chance = math.Rand(0,1)
		self.kids = 0
		self.maxKids = 20
		self.growthTime = 60 + math.random(0,30)
		--print(GLOBAL_coconut .. " global")
		if GLOBAL_coconut <= 15 then
			self.growthTime = 3 + math.random(0,4)
			self.maxKids = 2
		end

		--energy timer
		local timer_name = "treeCoco_" .. self:EntIndex()
		--print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
		--print(timer_name)
		--print("growthTime "..self.growthTime)
		--print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
		timer.Create(timer_name,self.growthTime,0, function() --every (some amount) seconds, update energy status
			--print("yeah, we doing it now!1")

			self.chance = math.Rand(0,1)
			--print(self.chance)
			if IsValid(self) then
				if self:IsOnFire() then
					--print("Remove coconut, fire " .. self:GetCreationID())
					self:Remove()
					--timer.Remove(timer_name)
				end
				if !self.isPlanted then
					if !self:OnGroundNotStupidEdition(self:GetPos()) && self.chance > 0.5 then
						--randomly make coconuts fall
						self:GetPhysicsObject():Wake()
					end
					if self.timer == 2 then
						--print("Remove coconut, time " .. self:GetCreationID())
						self:Remove()
						--timer.Remove(timer_name)
					end
					self.timer = self.timer + 1
				end
				if self.isPlanted && self.longTimer == 10 then
					--print("Remove coconut, long timer " .. self:GetCreationID())
					self:Remove()
					--timer.Remove(timer_name)
				elseif self.isPlanted then
					self.longTimer = self.longTimer + 1
				end
				--try to autoplant, otherwise if it is planted, upgrayed
				if !self.isPlanted then
					self:Upgrayed()
				elseif self.isPlanted && GLOBAL_coconut < GLOBAL_coconut_max then
					self:Upgrayed()
				end
			elseif !IsValid(self) then
				--print("Remove coconut, the timer itself " .. self:GetCreationID())
				timer.Remove(timer_name)
			end
		end)
		--print("puu Server: "..self:EntIndex())
	end
	--print("puu Client: "..self:EntIndex())
end

if SERVER then

	function ENT:OnRemove()
		GLOBAL_coconut = GLOBAL_coconut - 1
		--PrintTable( self:GetChildren() )
		if self.treeLevel >= 3 then
			--PRODUCE WOOD HERE
			puulle = ents.Create("micro_item_salainen_puulle")
			if ( !IsValid( puulle ) ) then return end
			puulle:SetPos(self:GetPos() + Vector(0,0,100))
			puulle:Spawn()

			--make a stump: kanto
			kanto = ents.Create("micro_item_salainen_kanto")
			if ( !IsValid( kanto ) ) then return end
			kanto:SetPos(self:GetPos() + Vector(0,0,0))
			kanto:Spawn()

			--make leaf gibs...
			--models/props_foliage/fern01.mdl
			--make leaves fall
			--leaf = ents.Create("prop_physics")
			--if ( !IsValid( leaf ) ) then return end
			--leaf:SetModel("models/props_foliage/fern01.mdl")
			--leaf:SetPos(self:GetPos() + Vector(0,0,128))
			--leaf:PhysicsInitStandard()
			--leaf:Spawn()
			--simple timer for like 2 seconds to destroy the new leaf(ves)
		end
	end
end --end SERVER

function ENT:Upgrayed()
	--plant the coconut
	--print("%%%%%%%%%%%%%%%%%%%%%%%%")
	--print(self.isPlanted)
	--print("treeLevel "..self.treeLevel)
	--print(self:OnGroundNotStupidEdition(self:GetPos()))
	--print("%%%%%%%%%%%%%%%%%%%%%%%%")

	if SERVER then
		if !self.isPlanted && self.treeLevel == 0 && self:OnGroundNotStupidEdition(self:GetPos()) && self:WaterLevel() == 0 then
			--autoplant section
			self.isPlanted = true
			--stop movement
			self:SetMoveType(0)
			--make sound
			--self:EmitSound("phx/eggcrack.wav")
			--go to next level
			self.treeLevel = self.treeLevel + 1 --1
			--move down to ground for the first time only
			self:SetPos(self:GetPos() + Vector(0,0,-9))
			--rotate the thing to be upwards
			self:SetAngles( Angle(self.angle1,self.angle2,0))
			--make sound
			self:EmitSound("phx/eggcrack.wav")
		elseif self.treeLevel == 1 then
			self:GetPhysicsObject():EnableCollisions(false)
			self:SetModel("models/props_foliage/urban_small_palm01.mdl")
			--PrintTable(self:GetMaterials())
			self:SetMaterial()
			self:PhysicsInitStandard()
			--self:SetMoveType(MOVETYPE_VPHYSICS)
			self:SetMoveType(0)
			self.treeLevel = self.treeLevel + 1 --2
			self.health = 250 * self.treeLevel

			--start scaling
			self:SetModelScale( 0.01, 0) --make the model really small to start
			self:SetModelScale( 1, self.growthTime) --grow the model over self.growthTime time. COOL!
		elseif self.treeLevel == 2 then
			self:GetPhysicsObject():EnableCollisions(false)
			self:SetModel("models/props_foliage/mall_palmplant02.mdl")
			--PrintTable(self:GetMaterials())
			self:SetMaterial()
			--print("bad material for leaves")
			self:PhysicsInitStandard()
			--self:SetMoveType(MOVETYPE_VPHYSICS)
			self:SetMoveType(0)
			self.treeLevel = self.treeLevel + 1 --3
			self.health = 250 * self.treeLevel

			--start scaling
			self:SetModelScale( 0.6, 0) --make the model small to start
			self:SetModelScale( 1.1, self.growthTime) --grow the model over self.growthTime time. COOL!
		elseif self.treeLevel == 3 then
			self:GetPhysicsObject():EnableCollisions(false)
			self:SetModel(self.lastModel[math.random(#self.lastModel)])

			self:SetMaterial()
			self:PhysicsInitStandard()
			--self:SetMoveType(MOVETYPE_VPHYSICS)
			self:SetMoveType(0)
			self.treeLevel = self.treeLevel + 1 --4
			self.health = 250 * self.treeLevel

			--start scaling
			self:SetModelScale( 0.8, 0) --make the model small to start
			self:SetModelScale( math.random(90, 120)/100, self.growthTime) --grow the model over self.growthTime time. COOL!
		elseif self.treeLevel == 4 && GLOBAL_coconut < 1.5*GLOBAL_coconut_max then
			for i = 1,math.random(1,self.numberOfFruitMax) do
				--timer.Simple( 3, function ()
				--print("making coconut again")
				newFruit = ents.Create("3p8_kookospahkina_puu")
				if ( !IsValid( newFruit ) ) then return end
				newFruit:SetPos(self:GetPos() + Vector(math.random(-12,12),math.random(-12,12),128+math.random(0,25)))
				--at position up in the tree (might need some if statements to do bearing fruit earlier/later)
					--maybe use the entity's height subtract some?
				newFruit:Spawn()
				--if self.chance >= 0.75 then
					--randomly make coconuts fall
					newFruit:GetPhysicsObject():Wake()
				--end
				self.kids = self.kids + 1
				--end)
			end
			if self.kids >= self.maxKids then
				--print("Remove coconut, kids " .. self:GetCreationID())
				self:Remove()
				--timer.Remove(timer_name)
			end
		end
	end
end

function ENT:OnGroundNotStupidEdition(position)
	local plantable = false
	local startpos = position
	local endpos = 	startpos + Vector(0,0,-1)*5
	local tmin = Vector(1,1,1)*-5
	local tmax = Vector(1,1,1)*5
	local tr = util.TraceHull( {
		start = startpos,
		endpos = endpos,
		filter = self,
		mins = tmin,
		maxs = tmax,
		--mask = MASK_ALL
	} )
	if not IsValid(tr.Entity) then
		tr = util.TraceLine({
			start = startpos,
			endpos = endpos,
			filter = self,
			--mask = MASK_ALL
		})
	end
	local ent = tr.Entity
	if IsValid(ent) && ent:GetClass() == "func_brush" then
		plantable = true
	end

	return plantable
end

function ENT:OnTakeDamage(damageto)
	self.health = self.health - damageto:GetDamage()
	if self.health <= 0 then
		self:EmitSound("weapons/debris1.wav")
		--print("Remove coconut, dmg " .. self:GetCreationID())
		self:Remove()
	end
end

--models/props_forest/log.mdl

--cocnut21
--0 models/hunter/misc/sphere025x025.mdl
	--models/props_pipes/guttermetal01a --material

--from tree thing on workshop
--1 models/props_foliage/ferns03.mdl
--2 models/props_foliage/ferns02.mdl
--3 models/props/de_dust/palm_tree_head_skybox.mdl
--4 models/props_foliage/urban_small_palm01.mdl
--5 models/props_foliage/mall_palmplant02.mdl
--6 models/props_foliage/mall_palm01_medium.mdl
--7 models/props_foliage/mall_palm01.mdl

--http://gamebanana.com/models/3622
--8 models/domitibingen/foliage/palms/d_palm_01/d_palm_s_07.mdl
--9 models/domitibingen/foliage/palms/d_palm_01/d_palm_b_01.mdl


text:
package matchers

import (
	"bytes"
	"encoding/json"
	"fmt"
	"reflect"
	"strings"

	"github.com/onsi/gomega/format"
)

type MatchJSONMatcher struct {
	JSONToMatch      interface{}
	firstFailurePath []interface{}
}

func (matcher *MatchJSONMatcher) Match(actual interface{}) (success bool, err error) {
	actualString, expectedString, err := matcher.prettyPrint(actual)
	if err != nil {
		return false, err
	}

	var aval interface{}
	var eval interface{}

	// this is guarded by prettyPrint
	json.Unmarshal([]byte(actualString), &aval)
	json.Unmarshal([]byte(expectedString), &eval)
	var equal bool
	equal, matcher.firstFailurePath = deepEqual(aval, eval)
	return equal, nil
}

func (matcher *MatchJSONMatcher) FailureMessage(actual interface{}) (message string) {
	actualString, expectedString, _ := matcher.prettyPrint(actual)
	return formattedMessage(format.Message(actualString, "to match JSON of", expectedString), matcher.firstFailurePath)
}

func (matcher *MatchJSONMatcher) NegatedFailureMessage(actual interface{}) (message string) {
	actualString, expectedString, _ := matcher.prettyPrint(actual)
	return formattedMessage(format.Message(actualString, "not to match JSON of", expectedString), matcher.firstFailurePath)
}

func formattedMessage(comparisonMessage string, failurePath []interface{}) string {
	var diffMessage string
	if len(failurePath) == 0 {
		diffMessage = ""
	} else {
		diffMessage = fmt.Sprintf("\n\nfirst mismatched key: %s", formattedFailurePath(failurePath))
	}
	return fmt.Sprintf("%s%s", comparisonMessage, diffMessage)
}

func formattedFailurePath(failurePath []interface{}) string {
	formattedPaths := []string{}
	for i := len(failurePath) - 1; i >= 0; i-- {
		switch p := failurePath[i].(type) {
		case int:
			formattedPaths = append(formattedPaths, fmt.Sprintf(`[%d]`, p))
		default:
			if i != len(failurePath)-1 {
				formattedPaths = append(formattedPaths, ".")
			}
			formattedPaths = append(formattedPaths, fmt.Sprintf(`"%s"`, p))
		}
	}
	return strings.Join(formattedPaths, "")
}

func (matcher *MatchJSONMatcher) prettyPrint(actual interface{}) (actualFormatted, expectedFormatted string, err error) {
	actualString, ok := toString(actual)
	if !ok {
		return "", "", fmt.Errorf("MatchJSONMatcher matcher requires a string, stringer, or []byte.  Got actual:\n%s", format.Object(actual, 1))
	}
	expectedString, ok := toString(matcher.JSONToMatch)
	if !ok {
		return "", "", fmt.Errorf("MatchJSONMatcher matcher requires a string, stringer, or []byte.  Got expected:\n%s", format.Object(matcher.JSONToMatch, 1))
	}

	abuf := new(bytes.Buffer)
	ebuf := new(bytes.Buffer)

	if err := json.Indent(abuf, []byte(actualString), "", "  "); err != nil {
		return "", "", fmt.Errorf("Actual '%s' should be valid JSON, but it is not.\nUnderlying error:%s", actualString, err)
	}

	if err := json.Indent(ebuf, []byte(expectedString), "", "  "); err != nil {
		return "", "", fmt.Errorf("Expected '%s' should be valid JSON, but it is not.\nUnderlying error:%s", expectedString, err)
	}

	return abuf.String(), ebuf.String(), nil
}

func deepEqual(a interface{}, b interface{}) (bool, []interface{}) {
	var errorPath []interface{}
	if reflect.TypeOf(a) != reflect.TypeOf(b) {
		return false, errorPath
	}

	switch a.(type) {
	case []interface{}:
		if len(a.([]interface{})) != len(b.([]interface{})) {
			return false, errorPath
		}

		for i, v := range a.([]interface{}) {
			elementEqual, keyPath := deepEqual(v, b.([]interface{})[i])
			if !elementEqual {
				return false, append(keyPath, i)
			}
		}
		return true, errorPath

	case map[string]interface{}:
		if len(a.(map[string]interface{})) != len(b.(map[string]interface{})) {
			return false, errorPath
		}

		for k, v1 := range a.(map[string]interface{}) {
			v2, ok := b.(map[string]interface{})[k]
			if !ok {
				return false, errorPath
			}
			elementEqual, keyPath := deepEqual(v1, v2)
			if !elementEqual {
				return false, append(keyPath, k)
			}
		}
		return true, errorPath

	default:
		return a == b, errorPath
	}
}