<|start_filename|>continuous_integration/docker/slurm/Dockerfile<|end_filename|>
FROM daskgateway/testing-base:latest

# Build and install slurm
RUN yum install -y epel-release \
    && yum install -y \
        bzip2 \
        epel-release \
        gcc \
        man2html \
        mariadb-server \
        mariadb-devel \
        munge \
        munge-devel \
        ncurses-devel \
        openssl \
        openssl-devel \
        perl \
        readline-devel \
    && curl -L https://download.schedmd.com/slurm/slurm-19.05.0.tar.bz2 -o /tmp/slurm.tar.bz2 \
    && cd /tmp \
    && tar -jxf /tmp/slurm.tar.bz2 \
    && cd slurm-19.05.0 \
    && ./configure --sysconfdir=/etc/slurm --with-mysql_config=/usr/bin --libdir=/usr/lib64 \
    && make install \
    && cd .. \
    && rm -rf slurm-19.05.0 \
    && rm -r slurm.tar.bz2 \
    && yum remove -y \
        bzip2 \
        gcc \
        mariadb-devel \
        munge-devel \
        ncurses-devel \
        openssl-devel \
        readline-devel \
    && yum clean all \
    && rm -rf /var/cache/yum

# Install supervisor and sudo
RUN yum install -y supervisor sudo \
    && yum clean all \
    && rm -rf /var/cache/yum

# Setup Slurm
RUN groupadd -r slurm  \
    && useradd -r -g slurm slurm \
    && mkdir /etc/sysconfig/slurm \
        /var/spool/slurmd \
        /var/run/slurmd \
        /var/lib/slurmd \
        /var/log/slurm \
    && chown slurm:slurm /var/spool/slurmd \
        /var/run/slurmd \
        /var/lib/slurmd \
        /var/log/slurm \
    && /sbin/create-munge-key

# Copy over files
COPY ./files /

RUN /root/init-mysql.sh

# Notify dask-gateway tests that Slurm is available
ENV TEST_DASK_GATEWAY_SLURM true

CMD ["/usr/bin/supervisord", "--configuration", "/etc/supervisord.conf"]


<|start_filename|>dask-gateway-server/dask-gateway-proxy/sni.go<|end_filename|>
package main

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"io"
	"net"
)

type tcpConn interface {
	net.Conn
	CloseWrite() error
	CloseRead() error
}

type peekedTCPConn struct {
	peeked []byte
	*net.TCPConn
}

func (c *peekedTCPConn) Read(p []byte) (n int, err error) {
	if len(c.peeked) > 0 {
		n = copy(p, c.peeked)
		c.peeked = c.peeked[n:]
		if len(c.peeked) == 0 {
			c.peeked = nil
		}
		return n, nil
	}
	return c.TCPConn.Read(p)
}

func wrapPeeked(inConn *net.TCPConn, br *bufio.Reader) tcpConn {
	peeked, _ := br.Peek(br.Buffered())
	return &peekedTCPConn{TCPConn: inConn, peeked: peeked}
}

type readonly struct {
	r io.Reader
	net.Conn
}

func (c readonly) Read(p []byte) (int, error) { return c.r.Read(p) }
func (readonly) Write(p []byte) (int, error)  { return 0, io.EOF }

func readSNI(inConn *net.TCPConn) (string, bool, tcpConn, error) {
	br := bufio.NewReader(inConn)
	hdr, err := br.Peek(1)
	if err != nil {
		return "", false, nil, err
	}

	if hdr[0] != 0x16 {
		// Not a TLS handshake
		return "", false, wrapPeeked(inConn, br), nil
	}

	const headerLen = 5
	hdr, err = br.Peek(headerLen)
	if err != nil {
		return "", false, wrapPeeked(inConn, br), nil
	}

	recLen := int(hdr[3])<<8 | int(hdr[4])
	helloBytes, err := br.Peek(headerLen + recLen)
	if err != nil {
		return "", true, wrapPeeked(inConn, br), nil
	}

	sni := ""
	server := tls.Server(readonly{r: bytes.NewReader(helloBytes)}, &tls.Config{
		GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) {
			sni = hello.ServerName
			return nil, nil
		},
	})
	server.Handshake()

	return sni, true, wrapPeeked(inConn, br), nil
}


<|start_filename|>dask-gateway-server/dask-gateway-proxy/main.go<|end_filename|>
package main

import (
	"bufio"
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

type RoutesMsg struct {
	Routes []Route
	Id     uint64
}

type EventsMsg struct {
	Events []Event
}

type Event struct {
	Id    uint64
	Type  string
	Route Route
}

type Route struct {
	Kind      string `json:"kind"`
	SNI       string `json:"sni,omitempty"`
	Path      string `json:"path,omitempty"`
	Target    string `json:"target"`
	targetURL *url.URL
}

func (r *Route) String() string {
	return fmt.Sprintf(
		"{kind: %q, sni: %q, path: %q, target: %q}",
		r.Kind, r.SNI, r.Path, r.Target,
	)
}

var errMissingTarget = errors.New("Must specify `target`")
var errMissingSNI = errors.New("Must specify `sni`")
var errMissingPath = errors.New("Must specify `path`")
var errUnknownKind = errors.New("Unknown route kind")
var errInvalidScheme = errors.New("Scheme must be `http` or `https`")

func (r *Route) Validate(full bool) error {
	if r.Kind == "SNI" {
		if r.SNI == "" {
			return errMissingSNI
		}
	} else if r.Kind == "PATH" {
		if r.Path == "" {
			return errMissingPath
		}
	} else {
		return errUnknownKind
	}
	if full {
		if r.Target == "" {
			return errMissingTarget
		}
		if r.Kind == "SNI" {
			r.Target = strings.TrimPrefix(r.Target, "tls://")
		} else {
			targetURL, err := url.Parse(r.Target)
			if err != nil {
				return err
			}
			if targetURL.Scheme != "http" && targetURL.Scheme != "https" {
				return errInvalidScheme
			}
			r.targetURL = targetURL
		}
	}
	return nil
}

type Proxy struct {
	logger     *Logger
	sniRoutes  map[string]string
	router     *Router
	routesLock sync.RWMutex
	proxy      *httputil.ReverseProxy
}

func NewProxy(logLevel LogLevel) *Proxy {
	out := Proxy{
		logger:    NewLogger("Proxy", logLevel),
		sniRoutes: make(map[string]string),
		router:    NewRouter(),
	}
	out.proxy = &httputil.ReverseProxy{
		Director:      out.director,
		FlushInterval: 250 * time.Millisecond,
		ErrorHandler:  out.errorHandler,
	}
	return &out
}

func awaitShutdown() {
	buf := make([]byte, 10)
	for {
		_, err := os.Stdin.Read(buf)
		if err == io.EOF {
			os.Exit(0)
		}
	}
}

func (p *Proxy) run(address, tcpAddress, apiURL, apiToken, tlsCert, tlsKey string, isChildProcess bool) {
	if isChildProcess {
		go awaitShutdown()
	}

	if tcpAddress == "" {
		tcpAddress = address
	}

	tlsListener, err := net.Listen("tcp", tcpAddress)
	if err != nil {
		p.logger.Errorf("%s", err)
		os.Exit(1)
	}
	p.logger.Infof("TCP Proxy serving at %s", tcpAddress)

	var httpListener net.Listener
	var forwarder *Forwarder = nil
	if address == tcpAddress {
		forwarder = newForwarder(tlsListener)
		httpListener = forwarder
	} else {
		httpListener, err = net.Listen("tcp", address)
		if err != nil {
			p.logger.Errorf("%s", err)
			os.Exit(1)
		}
	}
	p.logger.Infof("HTTP Proxy serving at %s", address)

	go p.runHTTP(httpListener, tlsCert, tlsKey)
	go p.runTCP(tlsListener, forwarder)
	p.watchRoutes(apiURL, apiToken, 15*time.Second)
}

func (p *Proxy) clearRoutes() {
	p.logger.Infof("Resetting routing table")
	p.sniRoutes = make(map[string]string)
	p.router = NewRouter()
}

func (p *Proxy) putRoute(route *Route) {
	if err := route.Validate(true); err != nil {
		p.logger.Warnf("Invalid route %s: %s", route, err)
		return
	}

	p.logger.Infof("Adding route %s", route)

	if route.Kind == "SNI" {
		p.sniRoutes[route.SNI] = route.Target
	} else {
		p.router.Put(route.Path, route.targetURL)
	}
}

func (p *Proxy) deleteRoute(route *Route) {
	if err := route.Validate(false); err != nil {
		p.logger.Warnf("Invalid route %s: %s", route, err)
		return
	}

	p.logger.Infof("Removing route %s", route)

	if route.Kind == "SNI" {
		delete(p.sniRoutes, route.SNI)
	} else {
		p.router.Delete(route.Path)
	}
}

func (p *Proxy) UpdateFromRoutes(routes []Route) {
	p.routesLock.Lock()
	defer p.routesLock.Unlock()
	p.clearRoutes()
	for _, route := range routes {
		p.putRoute(&route)
	}
	return
}

func (p *Proxy) UpdateFromEvents(events []Event) {
	p.routesLock.Lock()
	defer p.routesLock.Unlock()
	for _, event := range events {
		if event.Type == "PUT" {
			p.putRoute(&event.Route)
		} else if event.Type == "DELETE" {
			p.deleteRoute(&event.Route)
		} else {
			p.logger.Warnf("Unknown event type %s", event.Type)
		}
	}
}

// Tracker implementation
type apiClient struct {
	client   *http.Client
	apiURL   string
	apiToken string
}

func NewApiClient(apiURL string, apiToken string) *apiClient {
	return &apiClient{
		client: &http.Client{
			Timeout: 60 * time.Second,
		},
		apiURL:   apiURL,
		apiToken: apiToken,
	}
}

func (c *apiClient) getRoutingTable() (*RoutesMsg, error) {
	req, err := http.NewRequest("GET", c.apiURL, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Authorization", "token "+c.apiToken)

	resp, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode >= 400 {
		return nil, errors.New(resp.Status)
	}

	var msg RoutesMsg
	err = json.NewDecoder(resp.Body).Decode(&msg)
	return &msg, err
}

type evIter struct {
	body    io.ReadCloser
	scanner *bufio.Scanner
	events  []Event
	err     error
}

func NewEvIter(body io.ReadCloser) *evIter {
	return &evIter{
		body:    body,
		scanner: bufio.NewScanner(body),
	}
}

func (it *evIter) Next() bool {
	if ok := it.scanner.Scan(); !ok {
		it.body.Close()
		return false
	}
	var msg EventsMsg
	if err := json.Unmarshal(it.scanner.Bytes(), &msg); err != nil {
		it.err = err
		it.body.Close()
		return false
	}
	it.events = msg.Events
	return true
}

func (it *evIter) Err() error {
	return it.err
}

func (it *evIter) Events() []Event {
	return it.events
}

var errTooOld = errors.New("Routes `last_id` is too old, retry")

func (c *apiClient) watchRoutingEvents(lastId uint64) (*evIter, error) {
	url := c.apiURL + "?watch=1&last_id=" + strconv.FormatUint(lastId, 10)
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Authorization", "token "+c.apiToken)

	resp, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode >= 400 {
		resp.Body.Close()
		if resp.StatusCode == 410 {
			return nil, errTooOld
		}
		return nil, errors.New(resp.Status)
	}
	return NewEvIter(resp.Body), nil
}

func (p *Proxy) watchRoutes(apiURL, apiToken string, maxBackoff time.Duration) {
	p.logger.Infof("Watching %s for routing table updates...", apiURL)

	initialBackoff := 500 * time.Millisecond
	backoff := initialBackoff

	doBackoff := func(msg string, err error) {
		p.logger.Warnf(msg, backoff.Seconds(), err)
		time.Sleep(backoff)
		backoff = backoff * 2
		if backoff > maxBackoff {
			backoff = maxBackoff
		}
	}

	resetBackoff := func() {
		backoff = initialBackoff
	}

	client := NewApiClient(apiURL, apiToken)
	for {
		msg, err := client.getRoutingTable()
		if err != nil {
			doBackoff("Unexpected failure fetching routing table, retrying in %.1fs: %s", err)
			continue
		}
		resetBackoff()

		p.UpdateFromRoutes(msg.Routes)

		last_id := msg.Id

		for {
			it, err := client.watchRoutingEvents(last_id)
			if err == errTooOld {
				p.logger.Debugf("last_id is too old, fetching whole routing table")
				break
			} else if err != nil {
				doBackoff("Unexpected failure fetching routing events, retrying in %.1fs: %s", err)
				break
			}
			for it.Next() {
				events := it.Events()
				if len(events) > 0 {
					p.UpdateFromEvents(events)
					last_id = events[len(events)-1].Id
				}
			}
			if err := it.Err(); err != nil {
				doBackoff("Unexpected failure streaming routing events, retrying in %.1fs: %s", err)
				break
			}
		}
	}
}

// HTTP Proxy Implementation

func (p *Proxy) errorHandler(w http.ResponseWriter, r *http.Request, err error) {
	if err != context.Canceled {
		p.logger.Warnf("Web Proxy Error: %s", err)
	}
	w.WriteHeader(http.StatusBadGateway)
}

func (p *Proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	p.routesLock.RLock()
	target, path := p.router.Match(req.URL.Path)
	p.routesLock.RUnlock()

	var uri string
	if p.logger.level >= DEBUG {
		uri = req.URL.RequestURI()
	}

	if target != nil {
		targetQuery := target.RawQuery
		req.URL.Scheme = target.Scheme
		req.URL.Host = target.Host
		req.Host = target.Host
		req.URL.Path = singleJoiningSlash(target.Path, path)
		if targetQuery == "" || req.URL.RawQuery == "" {
			req.URL.RawQuery = targetQuery + req.URL.RawQuery
		} else {
			req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery
		}
		p.logger.Infof("%s -> %s", uri, req.URL)
		p.proxy.ServeHTTP(w, req)
	} else {
		p.logger.Infof("%s not found", uri)
		http.Error(w, "Not Found", http.StatusNotFound)
	}
}

func singleJoiningSlash(a, b string) string {
	aslash := strings.HasSuffix(a, "/")
	bslash := strings.HasPrefix(b, "/")
	switch {
	case aslash && bslash:
		return a + b[1:]
	case !aslash && !bslash:
		return a + "/" + b
	}
	return a + b
}

func (p *Proxy) director(req *http.Request) {
	// Request URL already modified above in ServeHTTP

	if _, ok := req.Header["User-Agent"]; !ok {
		// explicitly disable User-Agent so it's not set to default value
		req.Header.Set("User-Agent", "")
	}
}

func (p *Proxy) runHTTP(ln net.Listener, tlsCert, tlsKey string) {
	server := &http.Server{
		Handler: p,
	}
	var err error
	if tlsCert == "" {
		err = server.Serve(ln)
	} else {
		err = server.ServeTLS(ln, tlsCert, tlsKey)
	}
	if err != nil && err != http.ErrServerClosed {
		p.logger.Errorf("%s", err)
		os.Exit(1)
	}
}

// TCP proxy implementation

type Forwarder struct {
	net.Listener
	conns chan net.Conn
}

func newForwarder(ln net.Listener) *Forwarder {
	return &Forwarder{
		Listener: ln,
		conns:    make(chan net.Conn),
	}
}

func (h *Forwarder) Forward(conn net.Conn) {
	h.conns <- conn
}

func (h *Forwarder) Accept() (net.Conn, error) {
	conn := <-h.conns
	return conn, nil
}

func (p *Proxy) handleConnection(inConn *net.TCPConn, forwarder *Forwarder) {
	var err error

	sni, isTLS, pInConn, err := readSNI(inConn)
	if err != nil {
		p.logger.Debugf("Error extracting SNI: %s", err)
		inConn.Close()
		return
	}

	const sniPrefix = "daskgateway-"
	if !isTLS || !strings.HasPrefix(sni, sniPrefix) {
		if forwarder != nil {
			forwarder.Forward(pInConn)
			return
		} else {
			p.logger.Debug("Invalid connection attempt, closing")
			inConn.Close()
			return
		}
	}

	sni = sni[len(sniPrefix):]

	defer inConn.Close()

	p.routesLock.RLock()
	outAddr := p.sniRoutes[sni]
	p.routesLock.RUnlock()

	if outAddr == "" {
		p.logger.Infof("SNI %q not found", sni)
		return
	}

	p.logger.Infof("SNI %q -> %q", sni, outAddr)

	outConn, err := net.DialTimeout("tcp", outAddr, 10*time.Second)
	if err != nil {
		p.logger.Debugf("Failed to connect to destination %q: %s", outAddr, err)
		return
	}
	defer outConn.Close()

	var wg sync.WaitGroup
	wg.Add(2)
	go p.proxyConnections(&wg, pInConn, outConn.(*net.TCPConn))
	go p.proxyConnections(&wg, outConn.(*net.TCPConn), pInConn)
	wg.Wait()
}

func (p *Proxy) proxyConnections(wg *sync.WaitGroup, in, out tcpConn) {
	defer wg.Done()
	if _, err := io.Copy(in, out); err != nil {
		p.logger.Debugf("Error proxying %q -> %q: %s", in.RemoteAddr(), out.RemoteAddr(), err)
	}
	in.CloseRead()
	out.CloseWrite()
}

func (p *Proxy) runTCP(ln net.Listener, forwarder *Forwarder) {
	for {
		c, err := ln.Accept()
		if err != nil {
			p.logger.Debugf("Failed to accept new connection: %s", err)
		}

		go p.handleConnection(c.(*net.TCPConn), forwarder)
	}
}

func main() {
	var (
		address        string
		tcpAddress     string
		apiURL         string
		logLevelString string
		isChildProcess bool
		tlsCert        string
		tlsKey         string
	)

	command := flag.NewFlagSet("dask-gateway-proxy", flag.ExitOnError)
	command.StringVar(&address, "address", ":8000", "HTTP proxy listening address")
	command.StringVar(&tcpAddress, "tcp-address", "", "TCP proxy listening address. If empty, `address` will be used.")
	command.StringVar(&apiURL, "api-url", "", "The URL the proxy should watch for updating the routing table")
	command.StringVar(&logLevelString, "log-level", "INFO",
		"The log level. One of {DEBUG, INFO, WARN, ERROR}")
	command.BoolVar(&isChildProcess, "is-child-process", false,
		"If set, will exit when stdin EOFs. Useful when running as a child process.")
	command.StringVar(&tlsCert, "tls-cert", "", "TLS cert to use, if any.")
	command.StringVar(&tlsKey, "tls-key", "", "TLS key to use, if any.")

	command.Parse(os.Args[1:])

	token := os.Getenv("DASK_GATEWAY_PROXY_TOKEN")
	if token == "" {
		panic("DASK_GATEWAY_PROXY_TOKEN environment variable not set")
	}

	logLevel := ParseLevel(logLevelString)

	if (tlsCert == "") != (tlsKey == "") {
		panic("Both -tls-cert and -tls-key must be set to use HTTPS")
	}

	if apiURL == "" {
		panic("-api-url must be set")
	}
	if _, err := url.Parse(apiURL); err != nil {
		panic("Invalid -api-url: " + err.Error())
	}

	proxy := NewProxy(logLevel)

	proxy.run(address, tcpAddress, apiURL, token, tlsCert, tlsKey, isChildProcess)
}


<|start_filename|>dask-gateway-server/dask-gateway-proxy/logging.go<|end_filename|>
// Our tiny custom logging framework.
//
// Provides common log levels, and quick functions for formatting and writing
// output at those levels.
package main

import (
	"fmt"
	"io"
	"os"
	"sync"
	"time"
)

type LogLevel int8

const (
	ERROR LogLevel = iota - 1
	WARN
	INFO
	DEBUG
)

func ParseLevel(s string) LogLevel {
	switch s {
	case "ERROR", "error":
		return ERROR
	case "WARN", "warn":
		return WARN
	case "INFO", "info":
		return INFO
	case "DEBUG", "debug":
		return DEBUG
	}
	panic("Couldn't parse log level " + s)
}

func (l LogLevel) Char() byte {
	switch l {
	case ERROR:
		return 'E'
	case WARN:
		return 'W'
	case INFO:
		return 'I'
	case DEBUG:
		return 'D'
	}
	return '?'
}

type Logger struct {
	sync.Mutex
	name  string
	level LogLevel
	out   io.Writer
	buf   []byte
}

func NewLogger(name string, level LogLevel) *Logger {
	return &Logger{name: name, level: level, out: os.Stderr}
}

func (l *Logger) logMsg(level LogLevel, msg string) {
	if l.level >= level {
		now := time.Now() // get this early.
		l.Lock()
		defer l.Unlock()
		l.buf = l.buf[:0]
		l.buf = append(l.buf, '[')
		l.buf = append(l.buf, level.Char())
		l.buf = append(l.buf, ' ')
		l.buf = now.AppendFormat(l.buf, "2006-01-02 15:04:05.000")
		l.buf = append(l.buf, ' ')
		l.buf = append(l.buf, l.name...)
		l.buf = append(l.buf, "] "...)
		l.buf = append(l.buf, msg...)
		l.buf = append(l.buf, '\n')
		l.out.Write(l.buf)
	}
}

func (l *Logger) logF(level LogLevel, format string, args ...interface{}) {
	if l.level >= level {
		l.logMsg(level, fmt.Sprintf(format, args...))
	}
}

func (l *Logger) Debug(msg string) {
	l.logMsg(DEBUG, msg)
}

func (l *Logger) Debugf(format string, args ...interface{}) {
	l.logF(DEBUG, format, args...)
}

func (l *Logger) Info(msg string) {
	l.logMsg(INFO, msg)
}

func (l *Logger) Infof(format string, args ...interface{}) {
	l.logF(INFO, format, args...)
}

func (l *Logger) Warn(msg string) {
	l.logMsg(WARN, msg)
}

func (l *Logger) Warnf(format string, args ...interface{}) {
	l.logF(WARN, format, args...)
}

func (l *Logger) Error(msg string) {
	l.logMsg(ERROR, msg)
}

func (l *Logger) Errorf(format string, args ...interface{}) {
	l.logF(ERROR, format, args...)
}


<|start_filename|>dask-gateway-server/dask-gateway-proxy/fail_without_go_1_12.go<|end_filename|>
// +build !go1.12

package main

func compileTimeError() {
	GO_1_12_REQUIRED_FOR_BUILD
}


<|start_filename|>continuous_integration/docker/pbs/Dockerfile<|end_filename|>
FROM daskgateway/testing-base:latest

# Install pbspro
RUN yum install -y unzip \
    && curl -L -o /tmp/pbspro.zip https://github.com/PBSPro/pbspro/releases/download/v18.1.4/pbspro_1.8.4.centos7.zip \
    && unzip /tmp/pbspro.zip -d /tmp/pbspro \
    && yum install -y sudo /tmp/pbspro/pbspro*/pbspro-server-*.rpm \
    && yum remove -y unzip \
    && yum clean all \
    && rm -rf /var/cache/yum

# Notify dask-gateway tests that PBS is available
ENV TEST_DASK_GATEWAY_PBS true
ENV PBS_MASTER pbs_master

# Copy over files
COPY ./files /

ENTRYPOINT ["/opt/miniconda/bin/tini", "-g", "--"]
CMD ["/root/start.sh"]


<|start_filename|>continuous_integration/docker/base/Dockerfile<|end_filename|>
FROM centos:7.6.1810

# Install miniconda and tini
RUN yum install -y bzip2 \
    && curl https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh \
    && /bin/bash /tmp/miniconda.sh -b -p /opt/miniconda \
    && rm /tmp/miniconda.sh \
    && /opt/miniconda/bin/conda update conda -y \
    && /opt/miniconda/bin/conda config --set always_yes yes --set changeps1 no \
    && /opt/miniconda/bin/conda install tini \
    && /opt/miniconda/bin/conda clean -af \
    && find /opt/miniconda/ -type f -name '*.a' -delete \
    && find /opt/miniconda/ -type f -name '*.pyc' -delete \
    && yum remove -y bzip2 \
    && yum clean all \
    && rm -rf /var/cache/yum

# Install go
RUN curl https://dl.google.com/go/go1.12.2.linux-amd64.tar.gz -o /tmp/go.tar.gz \
    && tar -xzf /tmp/go.tar.gz -C /opt/ \
    && rm /tmp/go.tar.gz

# Make a few user accounts, and configure their bashrc files
RUN useradd -m dask \
    && useradd -m alice \
    && useradd -m bob \
    && groupadd dask_users \
    && usermod -a -G dask_users alice \
    && usermod -a -G dask_users bob \
    && echo 'export PATH="/opt/go/bin:/opt/miniconda/bin:$PATH"' >> ~/.bashrc \
    && echo 'export PATH="/opt/miniconda/bin:$PATH"' >> /home/dask/.bashrc \
    && echo 'export PATH="/opt/miniconda/bin:$PATH"' >> /home/alice/.bashrc \
    && echo 'export PATH="/opt/miniconda/bin:$PATH"' >> /home/bob/.bashrc


<|start_filename|>resources/helm/example-images/Dockerfile<|end_filename|>
# An example Dockerfile for creating an image to use as a dask scheduler/worker
# with dask-gateway.
#
# The only requirement for such an image is that it has a version of
# `dask-gateway` installed that is compatible with the corresponding
# `dask-gateway-server`.
#
# Additionally, it is recommended that your `dask-gateway` image uses an init
# process (we recommend `tini` https://github.com/krallin/tini). This isn't
# strictly required, but you may run into weird bugs without an init process.
#
# Note that the version of Python or Python distribution doesn't matter. Here
# we'll install python via miniconda, then use `conda` to install all
# dependencies
FROM debian:buster-slim

# The miniconda version and corresponding SHA256 to use.
# You can see all available options here:
# https://docs.conda.io/en/latest/miniconda_hashes.html
ARG CONDA_VERSION=py38_4.8.3
ARG CONDA_SHA256=879457af6a0bf5b34b48c12de31d4df0ee2f06a8e68768e5758c3293b2daf688

# Does the following in one layer:
# - Create user dask
# - Install tini
# - Install miniconda build dependencies
# - Download miniconda and check the sha256 checksum
# - Install miniconda
# - Configure conda to minimize automatic package updates
# - Remove all conda managed static libraries
# - Cleanup conda files
# - Uninstall miniconda build dependencies
RUN useradd -m -U -u 1000 dask \
    && apt-get update \
    && apt-get install -y tini wget bzip2 \
    && wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-$CONDA_VERSION-Linux-x86_64.sh \
    && echo "${CONDA_SHA256}  Miniconda3-$CONDA_VERSION-Linux-x86_64.sh" > miniconda.checksum \
    && if [ $(sha256sum -c miniconda.checksum | awk '{print $2}') != "OK" ] ; then exit 1; fi \
    && mv Miniconda3-$CONDA_VERSION-Linux-x86_64.sh miniconda.sh \
    && sh ./miniconda.sh -b -p /opt/conda \
    && rm miniconda.sh miniconda.checksum \
    && ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh \
    && echo ". /opt/conda/etc/profile.d/conda.sh" >> /home/dask/.profile \
    && echo "conda activate base" >> /home/dask/.profile \
    && echo "always_yes: true" >> /home/dask/.condarc \
    && echo "changeps1: false" >> /home/dask/.condarc \
    && echo "auto_update_conda: false" >> /home/dask/.condarc \
    && echo "aggressive_update_packages: []" >> /home/dask/.condarc \
    && find /opt/conda/ -follow -type f -name '*.a' -delete \
    && /opt/conda/bin/conda clean -afy \
    && chown -R dask:dask /opt/conda \
    && apt-get autoremove --purge -y wget bzip2 \
    && apt-get clean \
    && rm -rf /var/lib/apt/lists/*

# Setup the dask user's environment
USER 1000:1000
ENV PATH="/opt/conda/bin:$PATH"
WORKDIR /home/dask/

# Configure the entrypoint to use tini
ENTRYPOINT ["tini", "-g", "--"]

# Install dask-gateway.
# If you need additional libraries, you can add them here.
# We run `conda clean -af` afterwards to drop unnecessary files from the
# docker image. This helps reduce the image size.
RUN conda install -c conda-forge --freeze-installed \
        dask-gateway \
        && \
    conda clean -af


<|start_filename|>continuous_integration/docker/hadoop/Dockerfile<|end_filename|>
FROM daskgateway/testing-base:latest
MAINTAINER jcrist

# Install common utilities
RUN yum install -y epel-release \
    && yum install -y \
        sudo \
        bzip2 \
        java-1.8.0-openjdk \
        supervisor \
    && yum clean all \
    && rm -rf /var/cache/yum

# Install kerberos
RUN yum install -y \
        krb5-libs \
        krb5-server \
        krb5-workstation \
    && yum clean all \
    && rm -rf /var/cache/yum

# Install hadoop
RUN cd /etc/yum.repos.d/ && { curl -O https://archive.cloudera.com/cdh5/redhat/7/x86_64/cdh/cloudera-cdh5.repo ; cd -; } \
    && rpm --import https://archive.cloudera.com/cdh5/redhat/7/x86_64/cdh/RPM-GPG-KEY-cloudera \
    && yum install -y \
        hadoop-yarn-resourcemanager \
        hadoop-hdfs-namenode \
        hadoop-yarn-nodemanager \
        hadoop-hdfs-datanode \
        hadoop-client \
        hadoop-libhdfs \
    && yum clean all \
    && rm -rf /var/cache/yum

# Copy over files
COPY ./files /

# Fix container-executor permissions
RUN chmod 6050 /etc/hadoop/conf.kerberos/container-executor.cfg

# Configure and setup hadoop
RUN /root/setup-hadoop.sh

# Setup kerberos
RUN /root/setup-kerb.sh

ENV JAVA_HOME /usr/lib/jvm/jre-openjdk
ENV LIBHDFS3_CONF /etc/hadoop/conf/hdfs-site.xml
ENV HADOOP_CONF_DIR /etc/hadoop/conf
ENV HADOOP_HOME /usr/lib/hadoop
ENV HADOOP_COMMON_HOME /usr/lib/hadoop
ENV HADOOP_YARN_HOME /usr/lib/hadoop-yarn
ENV HADOOP_HDFS_HOME /usr/lib/hadoop-hdfs
ENV TEST_DASK_GATEWAY_YARN true

CMD ["/usr/bin/supervisord", "--configuration", "/etc/supervisord.conf"]
