text:
/**
*    Originally created by David Erickson, Stanford University
*
*    Licensed under the Apache License, Version 2.0 (the "License"); you may
*    not use this file except in compliance with the License. You may obtain
*    a copy of the License at
*
*         http://www.apache.org/licenses/LICENSE-2.0
*
*    Unless required by applicable law or agreed to in writing, software
*    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
*    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
*    License for the specific language governing permissions and limitations
*    under the License.
**/

package net.floodlightcontroller.core;

import java.net.SocketAddress;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Date;

import org.projectfloodlight.openflow.protocol.OFActionType;
import org.projectfloodlight.openflow.protocol.OFCapabilities;
import org.projectfloodlight.openflow.protocol.OFControllerRole;
import org.projectfloodlight.openflow.protocol.OFFactory;
import org.projectfloodlight.openflow.protocol.OFMessage;
import org.projectfloodlight.openflow.protocol.OFPortDesc;
import org.projectfloodlight.openflow.protocol.OFRequest;
import org.projectfloodlight.openflow.protocol.OFStatsReply;
import org.projectfloodlight.openflow.protocol.OFStatsRequest;
import org.projectfloodlight.openflow.types.DatapathId;
import org.projectfloodlight.openflow.types.OFPort;
import org.projectfloodlight.openflow.types.TableId;
import org.projectfloodlight.openflow.types.U64;

import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.ListenableFuture;

import net.floodlightcontroller.core.internal.OFConnection;
import net.floodlightcontroller.core.internal.TableFeatures;
import net.floodlightcontroller.core.web.serializers.IOFSwitchSerializer;

import com.fasterxml.jackson.databind.annotation.JsonSerialize;
/**
 * An openflow switch connecting to the controller.  This interface offers
 * methods for interacting with switches using OpenFlow, and retrieving
 * information about the switches.
 */
@JsonSerialize(using=IOFSwitchSerializer.class)
public interface IOFSwitch extends IOFMessageWriter {
    // Attribute keys
    // These match our YANG schema, make sure they are in sync
    public static final String SWITCH_DESCRIPTION_FUTURE = "description-future";
    public static final String SWITCH_DESCRIPTION_DATA = "description-data";
    public static final String SWITCH_SUPPORTS_NX_ROLE = "supports-nx-role";
    public static final String PROP_FASTWILDCARDS = "fast-wildcards";
    public static final String PROP_REQUIRES_L3_MATCH = "requires-l3-match";
    public static final String PROP_SUPPORTS_OFPP_TABLE = "supports-ofpp-table";
    public static final String PROP_SUPPORTS_OFPP_FLOOD = "supports-ofpp-flood";
    public static final String PROP_SUPPORTS_NETMASK_TBL = "supports-netmask-table";
    public static final String PROP_SUPPORTS_BSN_SET_TUNNEL_DST_ACTION =
            "supports-set-tunnel-dst-action";
    public static final String PROP_SUPPORTS_NX_TTL_DECREMENT = "supports-nx-ttl-decrement";

    public enum SwitchStatus {
       /** this switch is in the process of being handshaked. Initial State. */
       HANDSHAKE(false),
       /** the OF channel to this switch is currently in SLAVE role - the switch will not accept
        *  state-mutating messages from this controller node.
        */
       SLAVE(true),
       /** the OF channel to this switch is currently in MASTER (or EQUALS) role - the switch is
        *  controllable from this controller node.
        */
       MASTER(true),
       /** the switch has been sorted out and quarantined by the handshake. It does not show up
        *  in normal switch listings
        */
       QUARANTINED(false),
       /** the switch has disconnected, and will soon be removed from the switch database */
       DISCONNECTED(false);

       private final boolean visible;

       SwitchStatus(boolean visible) {
        this.visible = visible;
       }

       /** whether this switch is currently visible for normal operation */
       public boolean isVisible() {
            return visible;
       }

       /** whether this switch is currently ready to be controlled by this controller */
       public boolean isControllable() {
            return this == MASTER;
       }
    }

    SwitchStatus getStatus();

    /**
     * Returns switch features from features Reply
     * @return
     */
    long getBuffers();

    /**
     * Disconnect all the switch's channels and mark the switch as disconnected
     */
    void disconnect();

    Set<OFActionType> getActions();

    Set<OFCapabilities> getCapabilities();

    /**
     * Get the specific TableIds according to the ofp_table_features.
     * Not all switches have sequential TableIds, so this will give the
     * specific TableIds used by the switch.
     * @return
     */
    Collection<TableId> getTables();

    /**
     * @return a copy of the description statistics for this switch
     */
    SwitchDescription getSwitchDescription();

    /**
     * Get the IP address of the remote (switch) end of the connection
     * @return the inet address
     */
    SocketAddress getInetAddress();

    /**
     * Get list of all enabled ports. This will typically be different from
     * the list of ports in the OFFeaturesReply, since that one is a static
     * snapshot of the ports at the time the switch connected to the controller
     * whereas this port list also reflects the port status messages that have
     * been received.
     * @return Unmodifiable list of ports not backed by the underlying collection
     */
    Collection<OFPortDesc> getEnabledPorts();

    /**
     * Get list of the port numbers of all enabled ports. This will typically
     * be different from the list of ports in the OFFeaturesReply, since that
     * one is a static snapshot of the ports at the time the switch connected
     * to the controller whereas this port list also reflects the port status
     * messages that have been received.
     * @return Unmodifiable list of ports not backed by the underlying collection
     */
    Collection<OFPort> getEnabledPortNumbers();

    /**
     * Retrieve the port object by the port number. The port object
     * is the one that reflects the port status updates that have been
     * received, not the one from the features reply.
     * @param portNumber
     * @return port object
     */
    OFPortDesc getPort(OFPort portNumber);

    /**
     * Retrieve the port object by the port name. The port object
     * is the one that reflects the port status updates that have been
     * received, not the one from the features reply.
     * Port names are case insentive
     * @param portName
     * @return port object
     */
    OFPortDesc getPort(String portName);

    /**
     * Get list of all ports. This will typically be different from
     * the list of ports in the OFFeaturesReply, since that one is a static
     * snapshot of the ports at the time the switch connected to the controller
     * whereas this port list also reflects the port status messages that have
     * been received.
     * @return Unmodifiable list of ports
     */
    Collection<OFPortDesc> getPorts();

    /**
     * This is mainly for the benefit of the DB code which currently has the
     * requirement that list elements be sorted by key. Hopefully soon the
     * DB will handle the sorting automatically or not require it at all, in
     * which case we could get rid of this method.
     * @return
     */
    Collection<OFPortDesc> getSortedPorts();

    /**
     * @param portNumber
     * @return Whether a port is enabled per latest port status message
     * (not configured down nor link down nor in spanning tree blocking state)
     */
    boolean portEnabled(OFPort portNumber);

    /**
     * @param portNumber
     * @return Whether a port is enabled per latest port status message
     * (not configured down nor link down nor in spanning tree blocking state)
     */
    boolean portEnabled(String portName);

    /**
     * Check is switch is connected
     * @return Whether or not this switch is connected
     */
    boolean isConnected();

    /**
     * Retrieves the date the switch connected to this controller
     * @return the date
     */
    Date getConnectedSince();

    /**
     * Get the datapathId of the switch
     * @return
     */
    DatapathId getId();

    /**
     * Retrieves attributes of this switch
     * @return
     */
    Map<Object, Object> getAttributes();

    /**
     * Check if the switch is active. I.e., the switch is connected to this
     * controller and is in master role
     * @return
     */
    boolean isActive();

    /**
     * Get the current role of the controller for the switch
     * @return the role of the controller
     */
    OFControllerRole getControllerRole();

    /**
     * Checks if a specific switch property exists for this switch
     * @param name name of property
     * @return value for name
     */
    boolean hasAttribute(String name);

    /**
     * Set properties for switch specific behavior
     * @param name name of property
     * @return value for name
     */
    Object getAttribute(String name);

    /**
     * Check if the given attribute is present and if so whether it is equal
     * to "other"
     * @param name the name of the attribute to check
     * @param other the object to compare the attribute against.
     * @return true iff the specified attribute is set and equals() the given
     * other object.
     */
    boolean attributeEquals(String name, Object other);

    /**
     * Set properties for switch specific behavior
     * @param name name of property
     * @param value value for name
     */
    void setAttribute(String name, Object value);

    /**
     * Set properties for switch specific behavior
     * @param name name of property
     * @return current value for name or null (if not present)
     */
    Object removeAttribute(String name);

    /**
     * Returns a factory object that can be used to create OpenFlow messages.
     * @return
     */
    OFFactory getOFFactory();

    /**
     * Gets the OF connections for this switch instance
     * @return Collection of IOFConnection
     */
    ImmutableList<IOFConnection> getConnections();

    /**
     * Writes a message to the connection specified by the logical OFMessage category
     * @param m an OF Message
     * @param category the category of the OF Message to be sent
     * @return true upon success; false upon failure
     */
    boolean write(OFMessage m, LogicalOFMessageCategory category);

    /**
     * Writes a message list to the connection specified by the logical OFMessage category
     * @param msglist an OF Message list
     * @param category the category of the OF Message list to be sent
     * @return list of failed messages, if any; success denoted by empty list
     */
    Iterable<OFMessage> write(Iterable<OFMessage> msglist, LogicalOFMessageCategory category);

    /**
     * Get a connection specified by the logical OFMessage category
     * @param category the category for the connection the user desires
     * @return an OF Connection
     */
    OFConnection getConnectionByCategory(LogicalOFMessageCategory category);

    /** write a Stats (Multipart-) request, register for all corresponding reply messages.
     * Returns a Future object that can be used to retrieve the List of asynchronous
     * OFStatsReply messages when it is available.
     *
     * @param request stats request
     * @param category the category for the connection that this request should be written to
     * @return Future object wrapping OFStatsReply
     *         If the connection is not currently connected, will
     *         return a Future that immediately fails with a @link{SwitchDisconnectedException}.
     */
    <REPLY extends OFStatsReply> ListenableFuture<List<REPLY>> writeStatsRequest(OFStatsRequest<REPLY> request, LogicalOFMessageCategory category);

    /** write an OpenFlow Request message, register for a single corresponding reply message
     *  or error message.
     *
     * @param request
     * @param categiry the category for the connection that this request should be written to
     * @return a Future object that can be used to retrieve the asynchrounous
     *         response when available.
     *
     *         If the connection is not currently connected, will
     *         return a Future that immediately fails with a @link{SwitchDisconnectedException}.
     */
    <R extends OFMessage> ListenableFuture<R> writeRequest(OFRequest<R> request, LogicalOFMessageCategory category);

    /**
     * Get the features of a particular switch table. The features are cached from
     * the initial handshake, or, if applicable, from a more recent
     * OFTableFeaturesStatsRequest/Reply sent by a user module.
     *
     * @param table, The table of which to get features.
     * @return The table features or null if no features are known for the table requested.
     */
    public TableFeatures getTableFeatures(TableId table);

    /**
     * Get the number of tables as returned by the ofp_features_reply.
     * @return
     */
	short getNumTables();

	/**
	 * Get the one-way latency from the switch to the controller.
	 * @return milliseconds
	 */
	public U64 getLatency();
}


text:
package newsrover

import (
	"bufio"
	"bytes"
	"compress/flate"
	"crypto/tls"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/textproto"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

type RoverConfig struct {
	Host  string `json:"host"`
	SSL   bool   `json:"use_ssl"`
	Group string `json:"newsgroup"`

	AuthUser string `json:"auth_user"`
	AuthPass string `json:"auth_pass"`

	CheckEvery  int `json:"check_every"`
	FlushEvery  int `json:"flush_articles_every"`
	MaxArticles int `json:"max_buffered_articles"`

	StartAt  int    `json:"start_at_article"`
	StopAt   int    `json:"no_article_to_process"`
	Progress string `json:"progress"`
}

type Rover struct {
	conn   *textproto.Conn
	config RoverConfig

	reconnect func() (io.ReadWriteCloser, error)
	stop      chan bool

	processed     int
	processedLife int
	lastArticle   int
	finalArticle  int
	high          int

	serverSubject   int
	serverFrom      int
	serverDate      int
	serverMessageId int
	serverBytes     int
	serverMinXover  int

	progressFile *os.File
	articleNo    int

	sinks []Sink

	logger *log.Logger

	runningWg sync.WaitGroup
}

type FetchIter struct {
	zreader io.ReadCloser
	reader  *bufio.Reader
	err     error
	rover   *Rover
}

var ErrBadYenc error = fmt.Errorf("Only recieved bad Yencs.")

func NewRover(addr string, conf RoverConfig) (*Rover, error) {
	conn, err := net.Dial("tcp", addr)
	if err != nil {
		return nil, err
	}
	reconnect := func() (io.ReadWriteCloser, error) {
		return net.Dial("tcp", addr)
	}
	return newRover(conn, reconnect, conf)
}

func NewRoverSsl(addr string, conf RoverConfig) (*Rover, error) {
	conn, err := tls.Dial("tcp", addr, &tls.Config{InsecureSkipVerify: true})
	if err != nil {
		return nil, err
	}
	reconnect := func() (io.ReadWriteCloser, error) {
		return tls.Dial("tcp", addr, &tls.Config{InsecureSkipVerify: true})
	}
	return newRover(conn, reconnect, conf)
}

func newRover(conn io.ReadWriteCloser, reconnect func() (io.ReadWriteCloser, error), conf RoverConfig) (*Rover, error) {
	r := &Rover{
		config:    conf,
		reconnect: reconnect,
		sinks:     make([]Sink, 0, 2),
		logger:    log.New(ioutil.Discard, "", log.LstdFlags),
	}
	if err := r.strapConnection(conn); err != nil {
		return nil, err
	}
	if r.config.Progress != "" {
		var err error
		if r.progressFile, err = os.OpenFile(r.config.Progress, os.O_RDWR|os.O_CREATE, 0666); err != nil {
			fmt.Printf("WARNING: Progress File for group %s failed to open. (%s). Progress will not be saved.\n", conf.Group, err.Error())
		}
	}
	if r.config.CheckEvery == 0 {
		r.config.CheckEvery = 60
	}
	if r.config.FlushEvery == 0 {
		r.config.FlushEvery = 60
	}
	if r.config.MaxArticles == 0 {
		r.config.MaxArticles = 64
	}
	return r, nil
}

func (r *Rover) Group() string {
	return r.config.Group
}

func (r *Rover) strapConnection(conn io.ReadWriteCloser) error {
	r.conn = textproto.NewConn(conn)
	if _, _, err := r.conn.ReadCodeLine(200); err != nil {
		return err
	}
	if r.config.AuthUser != "" {
		if _, err := r.Do(281, "AUTHINFO USER %s", r.config.AuthUser); err != nil {
			if response, err := r.Do(381, "AUTHINFO USER %s", r.config.AuthUser); err == nil {
				if response, err := r.Do(281, "AUTHINFO PASS %s", r.config.AuthPass); err != nil {
					return fmt.Errorf("Authenication Error: %s (%s)", err.Error(), response)
				}
			} else {
				return fmt.Errorf("Authenication Error: %s (%s)", err.Error(), response)
			}
		}
	}
	if _, err := r.Do(215, "LIST OVERVIEW.FMT"); err != nil {
		return err
	}
	if lines, err := r.conn.ReadDotLines(); err == nil {
		for idx, line := range lines {
			line = strings.ToLower(line)
			line = strings.TrimSuffix(line, ":")
			switch line {
			case "subject":
				r.serverSubject = idx
				r.serverMinXover = idx + 1
			case "from":
				r.serverFrom = idx
				r.serverMinXover = idx + 1
			case "date":
				r.serverDate = idx
				r.serverMinXover = idx + 1
			case "message-id":
				r.serverMessageId = idx
				r.serverMinXover = idx + 1
			case "bytes", ":bytes":
				r.serverBytes = idx
				r.serverMinXover = idx + 1
			}
		}
	}
	return nil
}

func (r *Rover) SetLogger(logger *log.Logger) {
	if logger == nil {
		r.logger = log.New(ioutil.Discard, "", log.LstdFlags)
	} else {
		r.logger = logger
		if r.logger.Prefix() == "" {
			r.logger.SetPrefix("[NewsRover - " + r.config.Group + "]")
		}
	}
}

func parseN(s string) int {
	if b, err := strconv.Atoi(s); err == nil {
		return b
	}
	return 0
}

func (r *Rover) SwitchGroup(group string) (Group, error) {
	if response, err := r.Do(211, "GROUP %s", group); err == nil {
		values := strings.Fields(response)
		if len(values) != 4 {
			return Group{}, fmt.Errorf("Malformed response from GROUP function.")
		}
		if group != r.config.Group {
			if r.logger.Prefix() == "" || r.logger.Prefix() == "[NewsRover - "+r.config.Group+"]" {
				r.logger.SetPrefix("[NewsRover - " + group + "]")
			}
			r.config.Group = group
		}
		return Group{
			Name:   values[3],
			Number: parseN(values[0]),
			Low:    parseN(values[1]),
			High:   parseN(values[2]),
		}, nil
	} else {
		return Group{}, err
	}
}

func (r *Rover) Close() {
	r.conn.Close()
}

func (r *Rover) Do(expectCode int, format string, args ...interface{}) (string, error) {
	if err := r.conn.PrintfLine(format, args...); err != nil {
		return "", err
	}
	if _, response, err := r.conn.ReadCodeLine(expectCode); err == nil {
		return response, nil
	} else {
		return response, err
	}
}

func (r *Rover) fetch(articleId ...int) (int, error) {
	var command string
	var noa int
	switch len(articleId) {
	case 1:
		command = fmt.Sprintf("XZVER %d", articleId[0])
		noa = 1
	case 2:
		if articleId[0] == articleId[1] {
			command = fmt.Sprintf("XZVER %d", articleId[0])
			noa = 1
		} else if articleId[1] < 0 {
			command = fmt.Sprintf("XZVER %d-", articleId[0])
			noa = 64
		} else {
			command = fmt.Sprintf("XZVER %d-%d", articleId[0], articleId[1])
			noa = articleId[1] - articleId[0]
			if noa == 0 {
				noa = 1
			} else if noa < 0 {
				noa = noa * -1
			}
		}
	default:
		panic(fmt.Sprintf("Rover.Fetch called with an invalid number of arguments (Expected 1 or 2, got %d).", len(articleId)))
	}
	for i := 0; i < 3; i++ {
		if _, err := r.Do(224, command); err == nil {
			break
		} else if err == io.ErrClosedPipe {
			if newConn, err := r.reconnect(); err == nil {
				if err := r.strapConnection(newConn); err != nil {
					return 0, err
				}
				continue
			} else {
				return 0, err
			}
		} else {
			return 0, err
		}
	}
	return noa, nil
}

func (r *Rover) ParseArticle(line string) (Article, error) {
	var article Article
	fields := strings.Split(line, "\t")
	if len(fields)-1 < r.serverMinXover {
		return article, fmt.Errorf("Malformed line.")
	}
	article.Group = r.config.Group
	article.ArticleId = parseN(fields[0])
	for idx, field := range fields[1:] {
		switch idx {
		case r.serverSubject:
			article.Subject = field
		case r.serverFrom:
			article.From = field
		case r.serverDate:
			article.Date = field
		case r.serverMessageId:
			article.MessageId = field
		case r.serverBytes:
			article.Bytes = int64(parseN(field))
		}
	}
	return article, nil
}

func (r *Rover) Fetch(articleId ...int) ([]Article, error) {
	noa, e := r.fetch(articleId...)
	if e != nil {
		return nil, e
	}
	articles := make([]Article, 0, noa)

	dec := NewYencDecoder(r.conn.R)
	part, err := dec.Decode()
	if err != nil {
		return nil, err
	}
	r.conn.ReadDotBytes()
	zreader := flate.NewReader(bytes.NewReader(part.Body))
	defer zreader.Close()
	if err != nil {
		return nil, err
	}
	reader := bufio.NewReader(zreader)
	for {
		line, err := reader.ReadString('\n')
		if err != nil {
			if err == io.EOF {
				break
			}
			return nil, err
		}
		if article, err := r.ParseArticle(line); err == nil {
			articles = append(articles, article)
		}

	}
	return articles, nil
}

func (r *Rover) FetchIter(articleId ...int) (*FetchIter, error) {
	for i := 0; i < 3; i++ {
		if i > 0 {
			time.Sleep(5 * time.Duration(i) * time.Second)
		}
		if _, e := r.fetch(articleId...); e != nil {
			return nil, e
		}

		dec := NewYencDecoder(r.conn.R)
		part, err := dec.Decode()
		if err != nil {
			return nil, err
		}
		r.conn.ReadDotBytes()
		if bytes.Equal(part.Body, []byte{3, 0}) {
			r.logger.Println("Recieved bad yenc. Refetching...")
			continue
		}
		zreader := flate.NewReader(bytes.NewReader(part.Body))
		reader := bufio.NewReader(zreader)
		return &FetchIter{
			reader:  reader,
			zreader: zreader,
			err:     nil,
			rover:   r,
		}, nil
	}
	return nil, ErrBadYenc
}

func (i *FetchIter) Next(outArticle *Article) bool {
	for {
		line, err := i.reader.ReadString('\n')
		if err != nil {
			//i.rover.logger.Printf("Error reading article. (%s, %v, %d)", err.Error(), err == io.EOF, len(line))
			if err == io.EOF {
				break
			}
			i.err = err
			return false
		}
		if article, err := i.rover.ParseArticle(line); err == nil {
			*outArticle = article
			return true
		} else {
			i.rover.logger.Printf("Failed to parse line %s.", line)
		}
	}
	return false
}

func (i *FetchIter) Close() error {
	defer i.zreader.Close()
	if i.err != nil {
		return i.err
	}
	return nil
}

func (r *Rover) AddSink(sink Sink) {
	r.sinks = append(r.sinks, sink)
}

func (r *Rover) RemoveSink(sink Sink) {
	for idx, other := range r.sinks {
		if other == sink {
			r.sinks[idx] = r.sinks[len(r.sinks)-1]
			r.sinks = r.sinks[0 : len(r.sinks)-1]
			return
		}
	}
}

func (r *Rover) flush(articles []Article) {
	nr := make([]Article, len(articles))
	copy(nr, articles)
	r.logger.Printf("Flushing articles. Current Point: %d Processed: %d (%d) High: %d", r.lastArticle, r.processed, r.processedLife, r.high)
	for _, sink := range r.sinks {
		sink.Accept(nr)
	}
}

func (r *Rover) ReadProgress() int {
	if r.progressFile != nil {
		r.progressFile.Seek(0, 0)
		if d, err := ioutil.ReadAll(r.progressFile); err == nil {
			sp := strings.Split(string(d), ";")
			if n, err := strconv.Atoi(sp[0]); err == nil {
				if sp[1] == r.Group() {
					// reprocess some articles just to be safe
					if n-100 > 0 {
						return n - 100
					}
					return n
				}
			}
		}
	}
	return -1
}

func (r *Rover) SaveProgress(article int) {
	if r.progressFile != nil {
		r.progressFile.Seek(0, 0)
		r.progressFile.WriteString(fmt.Sprintf("%d;%s;", article, r.Group()))
	}
}

func (r *Rover) Serve() error {
	if group, err := r.SwitchGroup(r.config.Group); err == nil {
		if r.config.StartAt <= 0 {
			r.lastArticle = group.High + r.config.StartAt
		} else if r.config.StartAt > group.Low {
			r.lastArticle = r.config.StartAt
		} else {
			r.lastArticle = group.High
		}
	} else {
		r.logger.Println("An error occured switching newsgroup: ", err.Error())
		return err
	}
	progress := r.ReadProgress()
	if progress != -1 {
		r.lastArticle = progress
		r.logger.Printf("Starting rover from article %d.", r.lastArticle)
	}
	saveTime := 30 * time.Second
	flushTime := time.Duration(r.config.FlushEvery) * time.Second
	checkTime := time.Duration(r.config.CheckEvery) * time.Second
	flush := time.NewTimer(flushTime)
	check := time.NewTimer(1 * time.Millisecond)
	midCheck := 1
	var psave *time.Timer
	if r.progressFile != nil {
		psave = time.NewTimer(saveTime)
		midCheck++
	}
	getSaveChannel := func() <-chan time.Time {
		if r.progressFile != nil {
			return psave.C
		}
		return nil
	}
	bfSz := r.config.MaxArticles
	if bfSz < 0 {
		bfSz = 64
	}
	articleBuff := make([]Article, 0, bfSz+1)
	defer func() {
		if len(articleBuff) > 0 {
			r.flush(articleBuff)
		}
	}()
	defer func() {
		r.SaveProgress(r.lastArticle)
	}()
	r.runningWg.Add(1)
	defer func() {
		r.runningWg.Done()
	}()
	defer func() {
		r.stop = nil
	}()
	r.stop = make(chan bool)
	r.logger.Printf("Starting newsrover on group %s", r.config.Group)
	for {
		select {
		case <-r.stop:
			return nil
		case <-flush.C:
			if len(articleBuff) > 0 {
				r.flush(articleBuff)
				articleBuff = articleBuff[:0]
			}
			flush.Reset(flushTime)
		case <-getSaveChannel():
			r.SaveProgress(r.lastArticle)
			psave.Reset(saveTime)
		case <-check.C:
			if group, err := r.SwitchGroup(r.config.Group); err == nil {
				if group.High > r.lastArticle {
					r.logger.Printf("Reading new articles. Starting from %d going to %d", r.lastArticle, group.High)
					r.high = group.High
					r.processed = 0

					for start, end := r.lastArticle+1, r.lastArticle+1025; start <= group.High; {
						if end > group.High {
							end = group.High
						}
						//r.logger.Printf("Fetching range %d - %d", start, end)
						for k := 0; k < 3; k++ {
							if iter, err := r.FetchIter(start, end); err == nil {
								var article Article
								z := 0
								var maxArt int
								for iter.Next(&article) {
									r.processed++
									r.processedLife++
									articleBuff = append(articleBuff, article)
									if r.config.MaxArticles >= 0 {
										if len(articleBuff) > r.config.MaxArticles {
											r.flush(articleBuff)
											articleBuff = articleBuff[:0]
										}
									}
									if z%50 == 0 {
										for i := 0; i < midCheck; i++ {
											select {
											case <-flush.C:
												if len(articleBuff) > 0 {
													r.flush(articleBuff)
													articleBuff = articleBuff[:0]
												}
												flush.Reset(flushTime)
											case <-getSaveChannel():
												r.SaveProgress(article.ArticleId)
												psave.Reset(saveTime)
											default:
												i = 100
											}
										}
									}
									if article.ArticleId > maxArt {
										maxArt = article.ArticleId
									}
									//end = article.ArticleId
									z++
								}
								if z == 0 {
									r.logger.Printf("Was going to process articles, but no articles were actually processed.")
								}
								if err := iter.Close(); err != nil {
									r.logger.Printf("An error occured closing the iterator: ", err.Error())
									return err
								}
								if maxArt < end {
									r.logger.Printf("Wanted to read up to %d, only read up to %d.", end, maxArt)
									start = maxArt + 1
								} else {
									start = end + 1
								}
								end = start + 1024
								break
							} else if err == ErrBadYenc {
								if k == 2 {
									r.logger.Printf("[FATAL] Could not reliable get articles. Segments will be skipped.")
								} else {
									r.logger.Printf("Failed to fetch articles, badYenc.")
									r.logger.Printf("Sleeping for %d seconds...", (k+1)*30)
									time.Sleep(time.Duration(k+1) * 30 * time.Second)
									select {
									case <-flush.C:
										if len(articleBuff) > 0 {
											r.flush(articleBuff)
											articleBuff = articleBuff[:0]
										}
										flush.Reset(flushTime)
									default:
									}
								}
							} else {
								r.logger.Println("An error occured fetching the articles: ", err.Error())
								if newConn, err := r.reconnect(); err == nil {
									if err := r.strapConnection(newConn); err != nil {
										r.logger.Println("An error occured bootstrapping the connection: ", err.Error())
										return err
									}
									r.logger.Printf("Reconnected successfully.")
									r.SwitchGroup(r.config.Group)
									continue
								} else {
									r.logger.Println("An error reconnecting to the news server: ", err.Error())
									return err
								}
							}
						}
					}
					r.lastArticle = group.High
					if r.config.StopAt > 0 {
						if r.processedLife >= r.config.StopAt {
							r.logger.Printf("Successfully processed %d articles. Set to stop at %d. Stopping.", r.processedLife, r.config.StopAt)
							go func() {
								r.stop <- true
							}()
						}
					}
				}
			} else {
				r.logger.Printf("Error occured connecting to host: %s. Trying to reconnect.", err.Error())
				if newConn, err := r.reconnect(); err == nil {
					if err := r.strapConnection(newConn); err != nil {
						r.logger.Println("An error occured bootstrapping the connection: ", err.Error())
						return err
					}
					r.logger.Printf("Reconnected successfully.")
					check.Reset(1 * time.Millisecond)
					continue
				} else {
					r.logger.Println("An error reconnecting to the news server: ", err.Error())
					return err
				}
			}
			check.Reset(checkTime)
		}
	}
	return nil
}

func (r *Rover) Stop() {
	if r.stop != nil {
		r.stop <- true
		r.stop = nil
	}
	r.runningWg.Wait()
}

redpajama_set_name: RedPajamaGithub


text:
package org.pentaho.di.trans.steps.sapinput;

public class SapOutputField {
  private String sapFieldName;
  private SapType sapType;
  private String tableName;
  private String newName;
  private int targetType;

  /**
   * @param sapFieldName
   * @param sapType
   * @param tableName
   * @param newName
   * @param targetType
   */
  public SapOutputField( String sapFieldName, SapType sapType, String tableName, String newName, int targetType ) {
    this.sapFieldName = sapFieldName;
    this.sapType = sapType;
    this.tableName = tableName;
    this.newName = newName;
    this.targetType = targetType;
  }

  /**
   * @return the sapFieldName
   */
  public String getSapFieldName() {
    return sapFieldName;
  }

  /**
   * @param sapFieldName
   *          the sapFieldName to set
   */
  public void setSapFieldName( String sapFieldName ) {
    this.sapFieldName = sapFieldName;
  }

  /**
   * @return the sapType
   */
  public SapType getSapType() {
    return sapType;
  }

  /**
   * @param sapType
   *          the sapType to set
   */
  public void setSapType( SapType sapType ) {
    this.sapType = sapType;
  }

  /**
   * @return the tableName
   */
  public String getTableName() {
    return tableName;
  }

  /**
   * @param tableName
   *          the tableName to set
   */
  public void setTableName( String tableName ) {
    this.tableName = tableName;
  }

  /**
   * @return the newName
   */
  public String getNewName() {
    return newName;
  }

  /**
   * @param newName
   *          the newName to set
   */
  public void setNewName( String newName ) {
    this.newName = newName;
  }

  /**
   * @return the targetType
   */
  public int getTargetType() {
    return targetType;
  }

  /**
   * @param targetType
   *          the targetType to set
   */
  public void setTargetType( int targetType ) {
    this.targetType = targetType;
  }
}


text:
<?xml version="1.0" ?>
<!--
 | LICENSE: This file is part of the DITA Open Toolkit project hosted on
 |          Sourceforge.net. See the accompanying license.txt file for
 |          applicable licenses.
 *-->
<!--
 | (C) Copyright IBM Corporation 2006. All Rights Reserved.
 *-->
<catalog xmlns="urn:oasis:names:tc:entity:xmlns:xml:catalog"
         prefer="public">

  <public publicId="-//IBM//DTD DITA Message Reference//EN"
          uri="plugins/msgRef/dtd/msgRef.dtd"/>
  <public publicId="-//IBM//ELEMENTS DITA Message Reference//EN"
          uri="plugins/msgRef/dtd/msgRef.mod"/>

</catalog>


text:
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
        <target>System.out</target>
        <encoder>
            <pattern>%date{MM/dd HH:mm:ss:SSS} %-5level[%.15thread] %logger{1} %msg%n</pattern>
        </encoder>
    </appender>

    <logger name="scala.slick" level="INFO" />
    <logger name="com.zaxxer.hikari" level="INFO" />

    <root level="INFO">
        <appender-ref ref="CONSOLE"/>
    </root>
</configuration>


text:
using System;
using System.Xml.Serialization;

namespace HoloXPLOR.Data.DataForge
{
    [XmlRoot(ElementName = "ARModeSettings")]
    public partial class ARModeSettings
    {
        [XmlAttribute(AttributeName = "PlayerLabelOffsetX")]
        public Single PlayerLabelOffsetX { get; set; }

        [XmlAttribute(AttributeName = "PlayerLabelOffsetY")]
        public Single PlayerLabelOffsetY { get; set; }

        [XmlAttribute(AttributeName = "PlayerLabelOffsetZ")]
        public Single PlayerLabelOffsetZ { get; set; }

        [XmlAttribute(AttributeName = "QueryFarPlaneDistanceMax")]
        public Single QueryFarPlaneDistanceMax { get; set; }

        [XmlAttribute(AttributeName = "QueryRaycastsActiveMax")]
        public Int32 QueryRaycastsActiveMax { get; set; }

        [XmlAttribute(AttributeName = "QueryTimeBetweenCenterFocusRaycasts")]
        public Single QueryTimeBetweenCenterFocusRaycasts { get; set; }

        [XmlAttribute(AttributeName = "QueryTimeBetweenComponentRaycasts")]
        public Single QueryTimeBetweenComponentRaycasts { get; set; }

        [XmlAttribute(AttributeName = "QueryTimeBetweenEntityRaycasts")]
        public Single QueryTimeBetweenEntityRaycasts { get; set; }

        [XmlAttribute(AttributeName = "FocusTransitionTime")]
        public Single FocusTransitionTime { get; set; }

        [XmlAttribute(AttributeName = "FocusZoneHard_WidthPercentage")]
        public Single FocusZoneHard_WidthPercentage { get; set; }

        [XmlAttribute(AttributeName = "FocusZoneHard_HeightPercentage")]
        public Single FocusZoneHard_HeightPercentage { get; set; }

        [XmlAttribute(AttributeName = "FocusZoneMedium_WidthPercentage")]
        public Single FocusZoneMedium_WidthPercentage { get; set; }

        [XmlAttribute(AttributeName = "FocusZoneMedium_HeightPercentage")]
        public Single FocusZoneMedium_HeightPercentage { get; set; }

        [XmlAttribute(AttributeName = "ReticleOffsetPercentageX")]
        public Single ReticleOffsetPercentageX { get; set; }

        [XmlAttribute(AttributeName = "ReticleOffsetPercentageY")]
        public Single ReticleOffsetPercentageY { get; set; }

        [XmlAttribute(AttributeName = "PrototypeMode")]
        public ARModePrototypeMode PrototypeMode { get; set; }

        [XmlElement(ElementName = "PROTO_FloatingDynamic")]
        public ARModePrototypeSettings_FloatingDynamic PROTO_FloatingDynamic { get; set; }

        [XmlElement(ElementName = "PROTO_FloatingStatic")]
        public ARModePrototypeSettings_FloatingStatic PROTO_FloatingStatic { get; set; }

    }
}


text:
void get_hw_addr(char *interface, char *dest)
{
	char filename[100];
	sprintf(filename, "/sys/class/net/%s/address", interface);
	if(fopen(filename, "r") == NULL)
	{
		strcpy(dest, INTERFACE_NOT_FOUND);
		return;
	}
	FILE *info = fopen(filename, "r");

	char buffer[HWADDRLEN+2];

	fgets(buffer, HWADDRLEN, info);

	fclose(info);

	strcpy(dest, buffer);

}

int get_interface_speed(char *interface)
{
	char filename[100];
	sprintf(filename, "/sys/class/net/%s/speed", interface);
	if(fopen(filename, "r") == NULL)
	{
		return 0;
	}
	char buffer[10];
	FILE *info = fopen(filename, "r");

	fgets(buffer, 9, info);
	fclose(info);

	return atoi(buffer);

}

void get_interface_duplex(char *interface, char *dest)
{
	char filename[100];
	sprintf(filename, "/sys/class/net/%s/duplex", interface);
	if(fopen(filename, "r") == NULL)
	{
		strcpy(dest, INTERFACE_NOT_FOUND);
		return;
	}
	char buffer[20];
	FILE *info = fopen(filename, "r");

	fgets(buffer, 19, info);
	fclose(info);

	strcpy(dest, buffer);

}

bool is_loopback(char *interface)
{
	if ( strcmp(interface, "lo") == 0)
		return true;
	else
		return false;

}

long get_interface_t(char *interface)
{
	char filename[100];
	sprintf(filename, "/sys/class/net/%s/statistics/tx_bytes", interface);
	if(fopen(filename, "r") == NULL)
		return 0;
	char buffer[25];
	FILE *info = fopen(filename, "r");

	fgets(buffer, 24, info);
	fclose(info);

	return atol(buffer);

}

long get_interface_r(char *interface)
{
	char filename[100];
	sprintf(filename, "/sys/class/net/%s/statistics/rx_bytes", interface);
	if(fopen(filename, "r") == NULL)
		return 0;
	char buffer[25];
	FILE *info = fopen(filename, "r");

	fgets(buffer, 24, info);
	fclose(info);

	return atol(buffer);

}


text:
package com.ft.coolweather.gson;

import com.google.gson.annotations.SerializedName;

/**
 * Created by djhj2009 on 17/9/6.
 */

public class Forecast {
    public String date;
    @SerializedName("tmp")
    public Temperature temperature;
    @SerializedName("cond")
    public More more;

    public class Temperature {
        public String max;
        public String min;
    }

    public class More {

        @SerializedName("txt_d")
        public String info;

    }
}


text:
import json
import os
import shutil
import tempfile

from contextlib import contextmanager
from datetime import datetime, timedelta
from decimal import Decimal

from django.conf import settings
from django.core import mail

import mock
import pytest

from PIL import Image

from olympia import amo
from olympia.addons.models import Addon, AddonUser
from olympia.amo.templatetags.jinja_helpers import user_media_path
from olympia.amo.tests import (
    TestCase, addon_factory, user_factory, version_factory)
from olympia.amo.tests.test_helpers import get_addon_file, get_image_path
from olympia.amo.utils import image_size, utc_millesecs_from_epoch
from olympia.api.models import SYMMETRIC_JWT_TYPE, APIKey
from olympia.applications.models import AppVersion
from olympia.constants.base import VALIDATOR_SKELETON_RESULTS
from olympia.devhub import tasks
from olympia.files.models import FileUpload
from olympia.versions.models import Version

pytestmark = pytest.mark.django_db

def test_resize_icon_shrink():
    """ Image should be shrunk so that the longest side is 32px. """

    resize_size = 32
    final_size = (32, 12)

    _uploader(resize_size, final_size)

def test_resize_icon_enlarge():
    """ Image stays the same, since the new size is bigger than both sides. """

    resize_size = 350
    final_size = (339, 128)

    _uploader(resize_size, final_size)

def test_resize_icon_same():
    """ Image stays the same, since the new size is the same. """

    resize_size = 339
    final_size = (339, 128)

    _uploader(resize_size, final_size)

def test_resize_icon_list():
    """ Resize multiple images at once. """

    resize_size = [32, 339, 350]
    final_size = [(32, 12), (339, 128), (339, 128)]

    _uploader(resize_size, final_size)

def _uploader(resize_size, final_size):
    img = get_image_path('mozilla.png')
    original_size = (339, 128)

    src = tempfile.NamedTemporaryFile(
        mode='r+w+b', suffix='.png', delete=False, dir=settings.TMP_PATH)

    if not isinstance(final_size, list):
        final_size = [final_size]
        resize_size = [resize_size]
    uploadto = user_media_path('addon_icons')
    try:
        os.makedirs(uploadto)
    except OSError:
        pass
    for rsize, expected_size in zip(resize_size, final_size):
        # resize_icon moves the original
        shutil.copyfile(img, src.name)
        src_image = Image.open(src.name)
        assert src_image.size == original_size
        dest_name = os.path.join(uploadto, '1234')

        with mock.patch('olympia.amo.utils.pngcrush_image') as pngcrush_mock:
            return_value = tasks.resize_icon(src.name, dest_name, [rsize])
        dest_image = '%s-%s.png' % (dest_name, rsize)
        assert pngcrush_mock.call_count == 1
        assert pngcrush_mock.call_args_list[0][0][0] == dest_image
        assert image_size(dest_image) == expected_size
        # original should have been moved to -original
        orig_image = '%s-original.png' % dest_name
        assert os.path.exists(orig_image)

        # Return value of the task should be a dict with an icon_hash key
        # containing the 8 first chars of the md5 hash of the source file,
        # which is bb362450b00f0461c6bddc6b97b3c30b.
        assert return_value == {'icon_hash': 'bb362450'}

        os.remove(dest_image)
        assert not os.path.exists(dest_image)
        os.remove(orig_image)
        assert not os.path.exists(orig_image)
    shutil.rmtree(uploadto)

    assert not os.path.exists(src.name)

class ValidatorTestCase(TestCase):
    def setUp(self):
        # Because the validator calls dump_apps() once and then uses the json
        # file to find out which appversions are valid, all tests running the
        # validator need to create *all* possible appversions all tests using
        # this class might need.

        # 3.7a1pre is somehow required to exist by
        # amo-validator.
        # The other ones are app-versions we're using in our
        # tests.
        self.create_appversion('firefox', '2.0')
        self.create_appversion('firefox', '3.6')
        self.create_appversion('firefox', '3.7a1pre')
        self.create_appversion('firefox', '38.0a1')

        # Required for WebExtensions tests.
        self.create_appversion('firefox', '*')
        self.create_appversion('firefox', '42.0')
        self.create_appversion('firefox', '42.*')
        self.create_appversion('firefox', '43.0')

        # Required for 57-specific tests.
        self.create_appversion('android', '38.0a1')
        self.create_appversion('android', '*')
        self.create_appversion('firefox', '57.0')

        # Required for Thunderbird tests.
        self.create_appversion('thunderbird', '42.0')
        self.create_appversion('thunderbird', '45.0')

    def create_appversion(self, name, version):
        return AppVersion.objects.create(
            application=amo.APPS[name].id, version=version)

class TestValidator(ValidatorTestCase):
    mock_sign_addon_warning = json.dumps({
        "warnings": 1,
        "errors": 0,
        "messages": [
            {"context": None,
             "editors_only": False,
             "description": "Add-ons which are already signed will be "
                            "re-signed when published on AMO. This will "
                            "replace any existing signatures on the add-on.",
             "column": None,
             "type": "warning",
             "id": ["testcases_content", "signed_xpi"],
             "file": "",
             "tier": 2,
             "for_appversions": None,
             "message": "Package already signed",
             "uid": "87326f8f699f447e90b3d5a66a78513e",
             "line": None,
             "compatibility_type": None},
        ]
    })

    def setUp(self):
        super(TestValidator, self).setUp()
        self.upload = FileUpload.objects.create(
            path=get_addon_file('desktop.xpi'))
        assert not self.upload.valid

    def get_upload(self):
        return FileUpload.objects.get(pk=self.upload.pk)

    @mock.patch('olympia.devhub.tasks.run_validator')
    def test_pass_validation(self, _mock):
        _mock.return_value = '{"errors": 0}'
        tasks.validate(self.upload, listed=True)
        assert self.get_upload().valid

    @mock.patch('olympia.devhub.tasks.run_validator')
    def test_fail_validation(self, _mock):
        _mock.return_value = '{"errors": 2}'
        tasks.validate(self.upload, listed=True)
        assert not self.get_upload().valid

    @mock.patch('validator.submain.test_package')
    def test_validation_error(self, _mock):
        _mock.side_effect = Exception

        self.upload.update(path=get_addon_file('desktop.xpi'))

        assert self.upload.validation is None

        tasks.validate(self.upload, listed=True)
        self.upload.reload()
        validation = self.upload.processed_validation
        assert validation
        assert validation['errors'] == 1
        assert validation['messages'][0]['id'] == ['validator',
                                                   'unexpected_exception']
        assert not self.upload.valid

    @mock.patch('olympia.devhub.tasks.run_addons_linter')
    def test_validation_error_webextension(self, _mock):
        _mock.side_effect = Exception
        self.upload.update(path=get_addon_file('valid_webextension.xpi'))

        assert self.upload.validation is None

        tasks.validate(self.upload, listed=True)
        self.upload.reload()
        validation = self.upload.processed_validation
        assert validation
        assert validation['errors'] == 1
        assert validation['messages'][0]['id'] == [
            'validator', 'unexpected_exception']
        assert 'WebExtension' in validation['messages'][0]['message']
        assert not self.upload.valid

    @mock.patch('olympia.devhub.tasks.run_validator')
    def test_validation_signing_warning(self, _mock):
        """If we sign addons, warn on signed addon submission."""
        _mock.return_value = self.mock_sign_addon_warning
        tasks.validate(self.upload, listed=True)
        validation = json.loads(self.get_upload().validation)
        assert validation['warnings'] == 1
        assert len(validation['messages']) == 1

    @mock.patch('validator.validate.validate')
    @mock.patch('olympia.devhub.tasks.track_validation_stats')
    def test_track_validation_stats(self, mock_track, mock_validate):
        mock_validate.return_value = '{"errors": 0}'
        tasks.validate(self.upload, listed=True)
        mock_track.assert_called_with(mock_validate.return_value)

class TestMeasureValidationTime(TestValidator):

    def setUp(self):
        super(TestMeasureValidationTime, self).setUp()
        # Set created time back (just for sanity) otherwise the delta
        # would be in the microsecond range.
        self.upload.update(created=datetime.now() - timedelta(days=1))

    @contextmanager
    def statsd_timing_mock(self):
        statsd_calls = {}

        def capture_timing_call(metric, value):
            statsd_calls[metric] = value

        with mock.patch('olympia.devhub.tasks.statsd.timing') as mock_timing:
            mock_timing.side_effect = capture_timing_call
            yield statsd_calls

    def approximate_upload_time(self):
        upload_start = utc_millesecs_from_epoch(self.upload.created)
        now = utc_millesecs_from_epoch()
        return now - upload_start

    def assert_milleseconds_are_close(self, actual_ms, calculated_ms,
                                      fuzz=None):
        if fuzz is None:
            fuzz = Decimal(300)
        assert (actual_ms >= (calculated_ms - fuzz) and
                actual_ms <= (calculated_ms + fuzz))

    def handle_upload_validation_result(self,
                                        channel=amo.RELEASE_CHANNEL_LISTED):
        validation = amo.VALIDATOR_SKELETON_RESULTS.copy()
        tasks.handle_upload_validation_result(validation, self.upload.pk,
                                              channel, False)

    def test_track_upload_validation_results_time(self):
        with self.statsd_timing_mock() as statsd_calls:
            self.handle_upload_validation_result()

        rough_delta = self.approximate_upload_time()
        actual_delta = statsd_calls['devhub.validation_results_processed']
        self.assert_milleseconds_are_close(actual_delta, rough_delta)

    def test_track_upload_validation_results_with_file_size(self):
        with self.statsd_timing_mock() as statsd_calls:
            self.handle_upload_validation_result()

        # This test makes sure storage.size() works on a real file.
        rough_delta = self.approximate_upload_time()
        actual_delta = statsd_calls[
            'devhub.validation_results_processed_per_mb']
        # This value should not be scaled because this package is under 1MB.
        self.assert_milleseconds_are_close(actual_delta, rough_delta)

    def test_scale_large_xpi_times_per_megabyte(self):
        megabyte = Decimal(1024 * 1024)
        file_size_in_mb = Decimal(5)
        with mock.patch('olympia.devhub.tasks.storage.size') as mock_size:
            mock_size.return_value = file_size_in_mb * megabyte
            with self.statsd_timing_mock() as statsd_calls:
                self.handle_upload_validation_result()

        # Validation times for files larger than 1MB should be scaled.
        rough_delta = self.approximate_upload_time()
        rough_scaled_delta = Decimal(rough_delta) / file_size_in_mb
        actual_scaled_delta = statsd_calls[
            'devhub.validation_results_processed_per_mb']
        self.assert_milleseconds_are_close(actual_scaled_delta,
                                           rough_scaled_delta)

    def test_measure_small_files_in_separate_bucket(self):
        with mock.patch('olympia.devhub.tasks.storage.size') as mock_size:
            mock_size.return_value = 500  # less than 1MB
            with self.statsd_timing_mock() as statsd_calls:
                self.handle_upload_validation_result()

        rough_delta = self.approximate_upload_time()
        actual_delta = statsd_calls[
            'devhub.validation_results_processed_under_1mb']
        self.assert_milleseconds_are_close(actual_delta, rough_delta)

    def test_measure_large_files_in_separate_bucket(self):
        with mock.patch('olympia.devhub.tasks.storage.size') as mock_size:
            mock_size.return_value = (2014 * 1024) * 5  # 5MB
            with self.statsd_timing_mock() as statsd_calls:
                self.handle_upload_validation_result()

        rough_delta = self.approximate_upload_time()
        actual_delta = statsd_calls[
            'devhub.validation_results_processed_over_1mb']
        self.assert_milleseconds_are_close(actual_delta, rough_delta)

    def test_do_not_calculate_scaled_time_for_empty_files(self):
        with mock.patch('olympia.devhub.tasks.storage.size') as mock_size:
            mock_size.return_value = 0
            with self.statsd_timing_mock() as statsd_calls:
                self.handle_upload_validation_result()

        assert 'devhub.validation_results_processed_per_mb' not in statsd_calls

    def test_ignore_missing_upload_paths_for_now(self):
        with mock.patch('olympia.devhub.tasks.storage.exists') as mock_exists:
            mock_exists.return_value = False
            with self.statsd_timing_mock() as statsd_calls:
                self.handle_upload_validation_result()

        assert 'devhub.validation_results_processed' in statsd_calls
        assert 'devhub.validation_results_processed_per_mb' not in statsd_calls
        assert ('devhub.validation_results_processed_under_1mb' not in
                statsd_calls)

class TestTrackValidatorStats(TestCase):

    def setUp(self):
        super(TestTrackValidatorStats, self).setUp()
        patch = mock.patch('olympia.devhub.tasks.statsd.incr')
        self.mock_incr = patch.start()
        self.addCleanup(patch.stop)

    def result(self, **overrides):
        result = VALIDATOR_SKELETON_RESULTS.copy()
        result.update(overrides)
        return json.dumps(result)

    def test_count_all_successes(self):
        tasks.track_validation_stats(self.result(errors=0))
        self.mock_incr.assert_any_call(
            'devhub.validator.results.all.success'
        )

    def test_count_all_errors(self):
        tasks.track_validation_stats(self.result(errors=1))
        self.mock_incr.assert_any_call(
            'devhub.validator.results.all.failure'
        )

    def test_count_listed_results(self):
        tasks.track_validation_stats(self.result(metadata={'listed': True}))
        self.mock_incr.assert_any_call(
            'devhub.validator.results.listed.success'
        )

    def test_count_unlisted_results(self):
        tasks.track_validation_stats(self.result(metadata={'listed': False}))
        self.mock_incr.assert_any_call(
            'devhub.validator.results.unlisted.success'
        )

class TestRunAddonsLinter(ValidatorTestCase):

    def setUp(self):
        super(TestRunAddonsLinter, self).setUp()

        valid_path = get_addon_file('valid_webextension.xpi')
        invalid_path = get_addon_file('invalid_webextension_invalid_id.xpi')

        self.valid_upload = FileUpload.objects.create(path=valid_path)
        self.invalid_upload = FileUpload.objects.create(path=invalid_path)

    def get_upload(self, upload):
        return FileUpload.objects.get(pk=upload.pk)

    @mock.patch('olympia.devhub.tasks.run_addons_linter')
    def test_calls_run_linter(self, run_linter):
        run_linter.return_value = '{"errors": 0}'

        assert not self.valid_upload.valid

        tasks.validate(self.valid_upload, listed=True)

        upload = self.get_upload(self.valid_upload)
        assert upload.valid, upload.validation

    def test_run_linter_fail(self):
        tasks.validate(self.invalid_upload, listed=True)
        assert not self.get_upload(self.invalid_upload).valid

    def test_run_linter_path_doesnt_exist(self):
        with pytest.raises(ValueError) as exc:
            tasks.run_addons_linter('doesntexist')

        assert str(exc.value) == (
            'Path "doesntexist" is not a file or directory or '
            'does not exist.')

    def test_run_linter_use_temporary_file(self):
        TemporaryFile = tempfile.TemporaryFile

        with mock.patch('olympia.devhub.tasks.tempfile.TemporaryFile') as tmpf:
            tmpf.side_effect = lambda *a, **kw: TemporaryFile(*a, **kw)

            # This is a relatively small add-on (1.2M) but we are using
            # a temporary file for all our linter output.
            result = json.loads(tasks.run_addons_linter(
                get_addon_file('typo-gecko.xpi')
            ))

            assert tmpf.call_count == 2
            assert result['success']
            assert result['warnings'] == 24
            assert not result['errors']

class TestValidateFilePath(ValidatorTestCase):

    def test_amo_validator_success(self):
        result = tasks.validate_file_path(
            get_addon_file('valid_firefox_addon.xpi'),
            hash_=None, listed=True)
        assert result['success']
        assert not result['errors']
        assert not result['warnings']

    def test_amo_validator_fail_warning(self):
        result = tasks.validate_file_path(
            get_addon_file('invalid_firefox_addon_warning.xpi'),
            hash_=None, listed=True)
        assert not result['success']
        assert not result['errors']
        assert result['warnings']

    def test_amo_validator_fail_error(self):
        result = tasks.validate_file_path(
            get_addon_file('invalid_firefox_addon_error.xpi'),
            hash_=None, listed=True)
        assert not result['success']
        assert result['errors']
        assert not result['warnings']

    def test_amo_validator_addons_linter_success(self):
        result = tasks.validate_file_path(
            get_addon_file('valid_webextension.xpi'),
            hash_=None, listed=True, is_webextension=True)
        assert result['success']
        assert not result['errors']
        assert not result['warnings']

    def test_amo_validator_addons_linter_error(self):
        # This test assumes that `amo-validator` doesn't correctly
        # validate a invalid id in manifest.json
        result = tasks.validate_file_path(
            get_addon_file('invalid_webextension_invalid_id.xpi'),
            hash_=None, listed=True, is_webextension=True)
        assert not result['success']
        assert result['errors']
        assert not result['warnings']

class TestWebextensionIncompatibilities(ValidatorTestCase):
    fixtures = ['base/addon_3615']

    def setUp(self):
        self.addon = Addon.objects.get(pk=3615)

        # valid_webextension.xpi has version 1.0 so mock the original version
        self.addon.update(guid='beastify@mozilla.org')
        self.addon.current_version.update(version='0.9')
        self.update_files(
            version=self.addon.current_version,
            filename='delicious_bookmarks-2.1.106-fx.xpi')

    def update_files(self, **kw):
        for version in self.addon.versions.all():
            for file in version.files.all():
                file.update(**kw)

    def test_webextension_upgrade_is_annotated(self):
        assert all(f.is_webextension is False
                   for f in self.addon.current_version.all_files)

        file_ = get_addon_file('valid_webextension.xpi')
        upload = FileUpload.objects.create(path=file_, addon=self.addon)

        tasks.validate(upload, listed=True)

        upload.refresh_from_db()
        assert upload.processed_validation['is_upgrade_to_webextension']

        expected = ['validation', 'messages', 'webext_upgrade']
        assert upload.processed_validation['messages'][0]['id'] == expected
        assert upload.processed_validation['warnings'] == 1
        assert upload.valid

    def test_new_webextension_is_not_annotated(self):
        """https://github.com/mozilla/addons-server/issues/3679"""
        previous_file = self.addon.current_version.all_files[-1]
        previous_file.is_webextension = True
        previous_file.status = amo.STATUS_AWAITING_REVIEW
        previous_file.save()

        file_ = get_addon_file('valid_webextension.xpi')
        upload = FileUpload.objects.create(path=file_, addon=self.addon)

        tasks.validate(upload, listed=True)

        upload.refresh_from_db()
        validation = upload.processed_validation

        assert 'is_upgrade_to_webextension' not in validation
        expected = ['validation', 'messages', 'webext_upgrade']
        assert not any(msg['id'] == expected for msg in validation['messages'])
        assert validation['warnings'] == 0
        assert upload.valid

    def test_webextension_webext_to_webext_not_annotated(self):
        previous_file = self.addon.current_version.all_files[-1]
        previous_file.is_webextension = True
        previous_file.save()

        file_ = get_addon_file('valid_webextension.xpi')
        upload = FileUpload.objects.create(path=file_, addon=self.addon)

        tasks.validate(upload, listed=True)
        upload.refresh_from_db()

        validation = upload.processed_validation

        assert 'is_upgrade_to_webextension' not in validation
        expected = ['validation', 'messages', 'webext_upgrade']
        assert not any(msg['id'] == expected for msg in validation['messages'])
        assert validation['warnings'] == 0
        assert upload.valid

    def test_webextension_no_webext_no_warning(self):
        file_ = amo.tests.AMOPaths().file_fixture_path(
            'delicious_bookmarks-2.1.106-fx.xpi')
        upload = FileUpload.objects.create(path=file_, addon=self.addon)

        tasks.validate(upload, listed=True)
        upload.refresh_from_db()

        validation = upload.processed_validation

        assert 'is_upgrade_to_webextension' not in validation
        expected = ['validation', 'messages', 'webext_upgrade']
        assert not any(msg['id'] == expected for msg in validation['messages'])

    def test_webextension_cannot_be_downgraded(self):
        self.update_files(is_webextension=True)

        file_ = amo.tests.AMOPaths().file_fixture_path(
            'delicious_bookmarks-2.1.106-fx.xpi')
        upload = FileUpload.objects.create(path=file_, addon=self.addon)

        tasks.validate(upload, listed=True)
        upload.refresh_from_db()

        expected = ['validation', 'messages', 'webext_downgrade']
        validation = upload.processed_validation

        assert validation['messages'][0]['id'] == expected
        assert validation['messages'][0]['type'] == 'error'

    def test_webextension_downgrade_only_warning_unlisted(self):
        self.update_files(is_webextension=True)
        self.make_addon_unlisted(self.addon)

        file_ = amo.tests.AMOPaths().file_fixture_path(
            'delicious_bookmarks-2.1.106-fx.xpi')
        upload = FileUpload.objects.create(path=file_, addon=self.addon)

        tasks.validate(upload, listed=False)
        upload.refresh_from_db()

        expected = ['validation', 'messages', 'webext_downgrade']
        validation = upload.processed_validation

        assert validation['messages'][0]['id'] == expected
        assert validation['messages'][0]['type'] == 'warning'
        assert validation['errors'] == 0

    def test_webextension_cannot_be_downgraded_ignore_deleted_version(self):
        """Make sure even deleting the previous version does not prevent
        the downgrade error."""
        file_ = amo.tests.AMOPaths().file_fixture_path(
            'delicious_bookmarks-2.1.106-fx.xpi')

        self.update_files(is_webextension=True)

        deleted_version = version_factory(
            addon=self.addon, file_kw={'is_webextension': False})
        deleted_version.delete()

        upload = FileUpload.objects.create(path=file_, addon=self.addon)

        tasks.validate(upload, listed=True)
        upload.refresh_from_db()

        expected = ['validation', 'messages', 'webext_downgrade']

        validation = upload.processed_validation

        assert validation['messages'][0]['id'] == expected
        assert validation['messages'][0]['type'] == 'error'

    def test_no_upgrade_annotation_no_version(self):
        """Make sure there's no workaround the downgrade error."""
        self.addon.update(guid='guid@xpi')

        file_ = amo.tests.AMOPaths().file_fixture_path(
            'delicious_bookmarks-no-version.xpi')

        self.update_files(is_webextension=True)

        deleted_version = version_factory(
            addon=self.addon, file_kw={'is_webextension': False})
        deleted_version.delete()

        upload = FileUpload.objects.create(path=file_, addon=self.addon)
        upload.addon.version = None
        upload.addon.save()
        upload.save(update_fields=('version',))
        upload.refresh_from_db()

        tasks.validate(upload, listed=True)
        upload.refresh_from_db()

        expected = [u'testcases_installrdf', u'_test_rdf', u'missing_addon']

        validation = upload.processed_validation

        assert validation['messages'][0]['id'] == expected
        assert validation['messages'][0]['type'] == 'error'

class TestLegacyAddonRestrictions(ValidatorTestCase):
    def setUp(self):
        super(TestLegacyAddonRestrictions, self).setUp()

    def test_submit_legacy_addon_restricted(self):
        file_ = get_addon_file('valid_firefox_addon.xpi')
        upload = FileUpload.objects.create(path=file_)
        tasks.validate(upload, listed=True)

        upload.refresh_from_db()

        assert upload.processed_validation['errors'] == 1
        expected = ['validation', 'messages', 'legacy_addons_restricted']
        assert upload.processed_validation['messages'][0]['id'] == expected
        assert not upload.valid

    def test_submit_legacy_extension_not_a_new_addon(self):
        file_ = get_addon_file('valid_firefox_addon.xpi')
        addon = addon_factory(version_kw={'version': '0.1'})
        upload = FileUpload.objects.create(path=file_, addon=addon)
        tasks.validate(upload, listed=True)

        upload.refresh_from_db()

        assert upload.processed_validation['errors'] == 0
        assert upload.processed_validation['messages'] == []
        assert upload.valid

    def test_submit_legacy_extension_1st_version_in_that_channel(self):
        file_ = get_addon_file('valid_firefox_addon.xpi')
        addon = addon_factory(
            version_kw={'version': '0.1',
                        'channel': amo.RELEASE_CHANNEL_UNLISTED})
        upload = FileUpload.objects.create(path=file_, addon=addon)
        tasks.validate(upload, listed=True)

        upload.refresh_from_db()

        assert upload.processed_validation['errors'] == 1
        expected = ['validation', 'messages', 'legacy_addons_restricted']
        assert upload.processed_validation['messages'][0]['id'] == expected
        assert not upload.valid

    def test_submit_legacy_extension_1st_version_in_that_channel_reverse(self):
        file_ = get_addon_file('valid_firefox_addon.xpi')
        addon = addon_factory(
            version_kw={'version': '0.1',
                        'channel': amo.RELEASE_CHANNEL_LISTED})
        upload = FileUpload.objects.create(path=file_, addon=addon)
        tasks.validate(upload, listed=False)

        upload.refresh_from_db()

        assert upload.processed_validation['errors'] == 1
        expected = ['validation', 'messages', 'legacy_addons_restricted']
        assert upload.processed_validation['messages'][0]['id'] == expected
        assert not upload.valid

    def test_submit_webextension(self):
        file_ = get_addon_file('valid_webextension.xpi')
        upload = FileUpload.objects.create(path=file_)
        tasks.validate(upload, listed=True)

        upload.refresh_from_db()

        assert upload.processed_validation['errors'] == 0
        assert upload.processed_validation['messages'] == []
        assert upload.valid

    def test_submit_legacy_extension_targets_older_firefox_stricly(self):
        file_ = get_addon_file('valid_firefox_addon_strict_compatibility.xpi')
        upload = FileUpload.objects.create(path=file_)
        tasks.validate(upload, listed=True)

        upload.refresh_from_db()

        assert upload.processed_validation['errors'] == 0
        assert upload.processed_validation['messages'] == []
        assert upload.valid

    def test_submit_non_extension(self):
        file_ = get_addon_file('searchgeek-20090701.xml')
        upload = FileUpload.objects.create(path=file_)
        tasks.validate(upload, listed=True)

        upload.refresh_from_db()

        assert upload.processed_validation['errors'] == 0
        assert upload.processed_validation['messages'] == []
        assert upload.valid

    def test_submit_thunderbird_extension(self):
        file_ = get_addon_file('valid_firefox_and_thunderbird_addon.xpi')
        upload = FileUpload.objects.create(path=file_)
        tasks.validate(upload, listed=True)

        upload.refresh_from_db()

        assert upload.processed_validation['errors'] == 0
        assert upload.processed_validation['messages'] == []
        assert upload.valid

    def test_restrict_firefox_53_alpha(self):
        data = {
            'messages': [],
            'errors': 0,
            'detected_type': 'extension',
            'metadata': {
                'is_webextension': False,
                'is_extension': True,
                'strict_compatibility': True,
                'applications': {
                    'firefox': {
                        'max': '53a1'
                    }
                }
            }
        }
        results = tasks.annotate_legacy_addon_restrictions(
            data, is_new_upload=True)
        assert results['errors'] == 1
        assert len(results['messages']) > 0
        assert results['messages'][0]['id'] == [
            'validation', 'messages', 'legacy_addons_restricted']

    def test_restrict_themes(self):
        data = {
            'messages': [],
            'errors': 0,
            'detected_type': 'theme',
            'metadata': {
                'is_extension': False,
                'strict_compatibility': False,
                'applications': {
                    'firefox': {
                        'max': '54.0'
                    }
                }
            }
        }
        results = tasks.annotate_legacy_addon_restrictions(
            data, is_new_upload=True)
        assert results['errors'] == 1
        assert len(results['messages']) > 0
        assert results['messages'][0]['id'] == [
            'validation', 'messages', 'legacy_addons_restricted']

    def test_submit_legacy_upgrade(self):
        # Works because it's not targeting >= 57.
        file_ = get_addon_file('valid_firefox_addon.xpi')
        addon = addon_factory(version_kw={'version': '0.1'})
        upload = FileUpload.objects.create(path=file_, addon=addon)
        tasks.validate(upload, listed=True)

        upload.refresh_from_db()

        assert upload.processed_validation['errors'] == 0
        assert upload.processed_validation['messages'] == []
        assert upload.valid

    def test_submit_legacy_upgrade_targeting_firefox_57(self):
        # Should error since it's a legacy extension targeting 57.
        file_ = get_addon_file('valid_firefox_addon_targeting_57.xpi')
        addon = addon_factory(version_kw={'version': '0.1'})
        upload = FileUpload.objects.create(path=file_, addon=addon)
        tasks.validate(upload, listed=True)

        upload.refresh_from_db()

        assert upload.processed_validation['errors'] == 1
        assert len(upload.processed_validation['messages']) == 1
        assert upload.processed_validation['messages'][0]['type'] == 'error'
        assert upload.processed_validation['messages'][0]['id'] == [
            'validation', 'messages', 'legacy_addons_max_version']
        assert not upload.valid

    def test_submit_legacy_upgrade_targeting_57_strict_compatibility(self):
        # Should error just like if it didn't have strict compatibility, that
        # does not matter: it's a legacy extension, it should not target 57.
        file_ = get_addon_file(
            'valid_firefox_addon_targeting_57_strict_compatibility.xpi')
        addon = addon_factory(version_kw={'version': '0.1'})
        upload = FileUpload.objects.create(path=file_, addon=addon)
        tasks.validate(upload, listed=True)

        upload.refresh_from_db()

        assert upload.processed_validation['errors'] == 1
        assert len(upload.processed_validation['messages']) == 1
        assert upload.processed_validation['messages'][0]['type'] == 'error'
        assert upload.processed_validation['messages'][0]['id'] == [
            'validation', 'messages', 'legacy_addons_max_version']
        assert not upload.valid

    def test_submit_legacy_upgrade_targeting_star(self):
        # Should not error: extensions with a maxversion of '*' don't get the
        # error, the manifest parsing code will rewrite it as '56.*' instead.
        file_ = get_addon_file('valid_firefox_addon_targeting_star.xpi')
        addon = addon_factory(version_kw={'version': '0.1'})
        upload = FileUpload.objects.create(path=file_, addon=addon)
        tasks.validate(upload, listed=True)

        upload.refresh_from_db()

        assert upload.processed_validation['errors'] == 0
        assert upload.processed_validation['messages'] == []
        assert upload.valid

    def test_submit_webextension_upgrade_targeting_firefox_57(self):
        # Should not error: it's targeting 57 but it's a webextension.
        file_ = get_addon_file('valid_webextension_targeting_57.xpi')
        addon = addon_factory(version_kw={'version': '0.1'},
                              file_kw={'is_webextension': True})
        upload = FileUpload.objects.create(path=file_, addon=addon)
        tasks.validate(upload, listed=True)

        upload.refresh_from_db()

        assert upload.processed_validation['errors'] == 0
        messages = upload.processed_validation['messages']
        assert len(messages) == 1
        assert messages[0]['message'] == ('"strict_max_version" '
                                          'not required.')
        assert upload.valid

    def test_submit_dictionary_upgrade_targeting_firefox_57(self):
        # Should not error: non-extensions types are not affected by the
        # restriction, even if they target 57.
        file_ = get_addon_file('dictionary_targeting_57.xpi')
        addon = addon_factory(version_kw={'version': '0.1'},
                              type=amo.ADDON_DICT)
        upload = FileUpload.objects.create(path=file_, addon=addon)
        tasks.validate(upload, listed=True)

        upload.refresh_from_db()

        assert upload.processed_validation['errors'] == 0
        assert upload.processed_validation['messages'] == []
        assert upload.valid

    def test_submit_legacy_targeting_multiple_including_firefox_57(self):
        # By submitting a legacy extension targeting multiple apps, this add-on
        # avoids the restriction for new uploads, but it should still trigger
        # the one for legacy extensions targeting 57 or higher.
        data = {
            'messages': [],
            'errors': 0,
            'detected_type': 'extension',
            'metadata': {
                'is_webextension': False,
                'is_extension': True,
                'applications': {
                    'firefox': {
                        'max': '57.0'
                    },
                    'thunderbird': {
                        'max': '45.0'
                    }
                }
            }
        }
        results = tasks.annotate_legacy_addon_restrictions(
            data.copy(), is_new_upload=True)
        assert results['errors'] == 1
        assert len(results['messages']) > 0
        assert results['messages'][0]['id'] == [
            'validation', 'messages', 'legacy_addons_max_version']

        results = tasks.annotate_legacy_addon_restrictions(
            data.copy(), is_new_upload=False)
        assert results['errors'] == 1
        assert len(results['messages']) > 0
        assert results['messages'][0]['id'] == [
            'validation', 'messages', 'legacy_addons_max_version']

@mock.patch('olympia.devhub.tasks.send_html_mail_jinja')
def test_send_welcome_email(send_html_mail_jinja_mock):
    tasks.send_welcome_email(3615, ['del@icio.us'], {'omg': 'yes'})
    send_html_mail_jinja_mock.assert_called_with(
        ('Mozilla Add-ons: Your add-on has been submitted to'
         ' addons.mozilla.org!'),
        'devhub/email/submission.html',
        'devhub/email/submission.txt',
        {'omg': 'yes'},
        recipient_list=['del@icio.us'],
        from_email=settings.NOBODY_EMAIL,
        use_deny_list=False,
        perm_setting='individual_contact')

class TestSubmitFile(TestCase):
    fixtures = ['base/addon_3615']

    def setUp(self):
        super(TestSubmitFile, self).setUp()
        self.addon = Addon.objects.get(pk=3615)
        patcher = mock.patch('olympia.devhub.tasks.create_version_for_upload')
        self.create_version_for_upload = patcher.start()
        self.addCleanup(patcher.stop)

    def create_upload(self, version='1.0'):
        return FileUpload.objects.create(
            addon=self.addon, version=version, validation='{"errors":0}',
            automated_signing=False)

    @mock.patch('olympia.devhub.tasks.FileUpload.passed_all_validations', True)
    def test_file_passed_all_validations(self):
        upload = self.create_upload()
        tasks.submit_file(self.addon.pk, upload.pk, amo.RELEASE_CHANNEL_LISTED)
        self.create_version_for_upload.assert_called_with(
            self.addon, upload, amo.RELEASE_CHANNEL_LISTED)

    @mock.patch('olympia.devhub.tasks.FileUpload.passed_all_validations',
                False)
    def test_file_not_passed_all_validations(self):
        upload = self.create_upload()
        tasks.submit_file(self.addon.pk, upload.pk, amo.RELEASE_CHANNEL_LISTED)
        assert not self.create_version_for_upload.called

class TestCreateVersionForUpload(TestCase):
    fixtures = ['base/addon_3615']

    def setUp(self):
        super(TestCreateVersionForUpload, self).setUp()
        self.addon = Addon.objects.get(pk=3615)
        self.create_version_for_upload = (
            tasks.create_version_for_upload.non_atomic)
        self.mocks = {}
        for key in ['Version.from_upload', 'parse_addon']:
            patcher = mock.patch('olympia.devhub.tasks.%s' % key)
            self.mocks[key] = patcher.start()
            self.addCleanup(patcher.stop)
        self.user = user_factory()

    def create_upload(self, version='1.0'):
        return FileUpload.objects.create(
            addon=self.addon, version=version, user=self.user,
            validation='{"errors":0}', automated_signing=False)

    def test_file_passed_all_validations_not_most_recent(self):
        upload = self.create_upload()
        newer_upload = self.create_upload()
        newer_upload.update(created=datetime.today() + timedelta(hours=1))

        # Check that the older file won't turn into a Version.
        self.create_version_for_upload(self.addon, upload,
                                       amo.RELEASE_CHANNEL_LISTED)
        assert not self.mocks['Version.from_upload'].called

        # But the newer one will.
        self.create_version_for_upload(self.addon, newer_upload,
                                       amo.RELEASE_CHANNEL_LISTED)
        self.mocks['Version.from_upload'].assert_called_with(
            newer_upload, self.addon, [amo.PLATFORM_ALL.id],
            amo.RELEASE_CHANNEL_LISTED,
            parsed_data=self.mocks['parse_addon'].return_value)

    def test_file_passed_all_validations_version_exists(self):
        upload = self.create_upload()
        Version.objects.create(addon=upload.addon, version=upload.version)

        # Check that the older file won't turn into a Version.
        self.create_version_for_upload(self.addon, upload,
                                       amo.RELEASE_CHANNEL_LISTED)
        assert not self.mocks['Version.from_upload'].called

    def test_file_passed_all_validations_most_recent_failed(self):
        upload = self.create_upload()
        newer_upload = self.create_upload()
        newer_upload.update(created=datetime.today() + timedelta(hours=1),
                            valid=False,
                            validation=json.dumps({"errors": 5}))

        self.create_version_for_upload(self.addon, upload,
                                       amo.RELEASE_CHANNEL_LISTED)
        assert not self.mocks['Version.from_upload'].called

    def test_file_passed_all_validations_most_recent(self):
        upload = self.create_upload(version='1.0')
        newer_upload = self.create_upload(version='0.5')
        newer_upload.update(created=datetime.today() + timedelta(hours=1))

        # The Version is created because the newer upload is for a different
        # version_string.
        self.create_version_for_upload(self.addon, upload,
                                       amo.RELEASE_CHANNEL_LISTED)
        self.mocks['parse_addon'].assert_called_with(
            upload, self.addon, user=self.user)
        self.mocks['Version.from_upload'].assert_called_with(
            upload, self.addon, [amo.PLATFORM_ALL.id],
            amo.RELEASE_CHANNEL_LISTED,
            parsed_data=self.mocks['parse_addon'].return_value)

    def test_file_passed_all_validations_beta_string(self):
        upload = self.create_upload(version='1.0-beta1')
        self.create_version_for_upload(self.addon, upload,
                                       amo.RELEASE_CHANNEL_LISTED)
        self.mocks['parse_addon'].assert_called_with(
            upload, self.addon, user=self.user)
        self.mocks['Version.from_upload'].assert_called_with(
            upload, self.addon, [amo.PLATFORM_ALL.id],
            amo.RELEASE_CHANNEL_LISTED,
            parsed_data=self.mocks['parse_addon'].return_value)

    def test_file_passed_all_validations_no_version(self):
        upload = self.create_upload(version=None)
        self.create_version_for_upload(self.addon, upload,
                                       amo.RELEASE_CHANNEL_LISTED)
        self.mocks['parse_addon'].assert_called_with(
            upload, self.addon, user=self.user)
        self.mocks['Version.from_upload'].assert_called_with(
            upload, self.addon, [amo.PLATFORM_ALL.id],
            amo.RELEASE_CHANNEL_LISTED,
            parsed_data=self.mocks['parse_addon'].return_value)

class TestAPIKeyInSubmission(TestCase):

    def setUp(self):
        self.user = user_factory()

        s = '656b16a8ab71686fcfcd04d574bc28be9a1d8252141f54cfb5041709262b84f4'
        self.key = APIKey.objects.create(
            user=self.user,
            type=SYMMETRIC_JWT_TYPE,
            key='user:12345:678',
            secret=s)
        self.addon = addon_factory(users=[self.user],
                                   version_kw={'version': '0.1'},
                                   file_kw={'is_webextension': True})
        self.file = get_addon_file('webextension_containing_api_key.xpi')

    def test_api_key_in_new_submission_is_found(self):
        upload = FileUpload.objects.create(path=self.file, user=self.user)
        tasks.validate(upload, listed=True)

        upload.refresh_from_db()

        assert upload.processed_validation['errors'] == 1
        messages = upload.processed_validation['messages']
        assert len(messages) == 1
        assert messages[0]['id'] == [
            u'validation', u'messages', u'api_key_detected']
        assert ('Your developer API key was found in the submitted '
                'file.' in messages[0]['message'])
        assert not upload.valid

        # If the key has been revoked, there is no active key,
        # so `get_jwt_key` raises `DoesNotExist`.
        with pytest.raises(APIKey.DoesNotExist):
            APIKey.get_jwt_key(user_id=self.user.id)

        assert len(mail.outbox) == 1
        assert ('Your AMO API credentials have been revoked'
                in mail.outbox[0].subject)
        assert mail.outbox[0].to[0] == self.user.email

    def test_api_key_in_submission_is_found(self):
        upload = FileUpload.objects.create(path=self.file, addon=self.addon,
                                           user=self.user)
        tasks.validate(upload, listed=True)

        upload.refresh_from_db()

        assert upload.processed_validation['errors'] == 1
        messages = upload.processed_validation['messages']
        assert len(messages) == 1
        assert messages[0]['id'] == [
            u'validation', u'messages', u'api_key_detected']
        assert ('Your developer API key was found in the submitted '
                'file.' in messages[0]['message'])
        assert not upload.valid

        # If the key has been revoked, there is no active key,
        # so `get_jwt_key` raises `DoesNotExist`.
        with pytest.raises(APIKey.DoesNotExist):
            APIKey.get_jwt_key(user_id=self.user.id)

        assert len(mail.outbox) == 1
        assert ('Your AMO API credentials have been revoked'
                in mail.outbox[0].subject)
        assert ('never share your credentials' in mail.outbox[0].body)
        assert mail.outbox[0].to[0] == self.user.email

    def test_coauthor_api_key_in_submission_is_found(self):
        coauthor = user_factory()
        AddonUser.objects.create(addon=self.addon, user_id=coauthor.id)
        upload = FileUpload.objects.create(path=self.file, addon=self.addon,
                                           user=coauthor)
        tasks.validate(upload, listed=True)

        upload.refresh_from_db()

        assert upload.processed_validation['errors'] == 1
        messages = upload.processed_validation['messages']
        assert len(messages) == 1
        assert messages[0]['id'] == [
            u'validation', u'messages', u'api_key_detected']
        assert ('The developer API key of a coauthor was found in the '
                'submitted file.' in messages[0]['message'])
        assert not upload.valid

        # If the key has been revoked, there is no active key,
        # so `get_jwt_key` raises `DoesNotExist`.
        with pytest.raises(APIKey.DoesNotExist):
            APIKey.get_jwt_key(user_id=self.user.id)

        assert len(mail.outbox) == 1
        assert ('Your AMO API credentials have been revoked'
                in mail.outbox[0].subject)
        assert ('never share your credentials' in mail.outbox[0].body)
        # We submit as the coauthor, the leaked key is the one from 'self.user'
        assert mail.outbox[0].to[0] == self.user.email

    def test_api_key_already_revoked_by_developer(self):
        self.key.update(is_active=None)
        tasks.revoke_api_key(self.key.id)
        # If the key has already been revoked, there is no active key,
        # so `get_jwt_key` raises `DoesNotExist`.
        with pytest.raises(APIKey.DoesNotExist):
            APIKey.get_jwt_key(user_id=self.user.id)

    def test_api_key_already_regenerated_by_developer(self):
        self.key.update(is_active=None)
        current_key = APIKey.new_jwt_credentials(user=self.user)
        tasks.revoke_api_key(self.key.id)
        key_from_db = APIKey.get_jwt_key(user_id=self.user.id)
        assert current_key.key == key_from_db.key
        assert current_key.secret == key_from_db.secret

    def test_revoke_task_is_called(self):
        mock_str = 'olympia.devhub.tasks.revoke_api_key'
        wrapped = tasks.revoke_api_key
        with mock.patch(mock_str, wraps=wrapped) as mock_revoke:
            upload = FileUpload.objects.create(path=self.file, user=self.user)
            tasks.validate(upload, listed=True)
            upload.refresh_from_db()
            mock_revoke.apply_async.assert_called_with(
                kwargs={'key_id': self.key.id}, countdown=120)

        assert not upload.valid

    def test_does_not_revoke_for_different_author(self):
        different_author = user_factory()
        upload = FileUpload.objects.create(path=self.file,
                                           user=different_author)
        tasks.validate(upload, listed=True)

        upload.refresh_from_db()

        assert upload.processed_validation['errors'] == 0
        assert upload.valid

    def test_does_not_revoke_safe_webextension(self):
        file_ = get_addon_file('valid_webextension.xpi')
        upload = FileUpload.objects.create(path=file_, user=self.user)
        tasks.validate(upload, listed=True)

        upload.refresh_from_db()

        assert upload.processed_validation['errors'] == 0
        assert upload.processed_validation['messages'] == []
        assert upload.valid

    def test_validation_finishes_if_containing_binary_content(self):
        file_ = get_addon_file('webextension_containing_binary_files.xpi')
        upload = FileUpload.objects.create(path=file_, user=self.user)
        tasks.validate(upload, listed=True)

        upload.refresh_from_db()

        assert upload.processed_validation['errors'] == 0
        assert upload.processed_validation['messages'] == []
        assert upload.valid

    def test_validation_finishes_if_containing_invalid_filename(self):
        file_ = get_addon_file('invalid_webextension.xpi')
        upload = FileUpload.objects.create(path=file_, user=self.user)
        tasks.validate(upload, listed=True)

        upload.refresh_from_db()

        # https://github.com/mozilla/addons-server/issues/8208
        # causes this to be 2 (and invalid) instead of 0 (and valid).
        # The invalid filename error is caught and raised outside of this
        # validation task.
        assert upload.processed_validation['errors'] == 2
        assert not upload.valid


text:
<?php

namespace Exposure\Model;

use Sociable\Model\MultiCurrencyValue,
    Sociable\Model\MultiLanguageString,
    Sociable\Utility\StringValidator;

class SponsorReturnOnFinancialContribution {
    protected $id = null;

    /** @var Theme */
    protected $theme = null;
    const EXCEPTION_INVALID_THEME = 'invalid theme';

    /** @var MultiCurrencyValue */
    protected $amount = null;
    const EXCEPTION_INVALID_AMOUNT = 'invalid amount';

    /** @var MultiLanguageString */
    protected $description = null;
    const EXCEPTION_INVALID_DESCRIPTION = 'invalid description';

    /** @var SponsorReturnType */
    protected $type = null;
    const EXCEPTION_INVALID_TYPE = 'invalid type';

    public function getId() {
        return $this->id;
    }

    public function getTheme() {
        return $this->theme;
    }

    public function setTheme(Theme $theme = null) {
        $this->theme = $theme;
        return $this->theme;
    }

    public function setDescription(MultiLanguageString $description) {
        try {
            $this->validateReturnDescription($description);
        }
        catch (Exception $e) {
            $this->description = null;
            throw $e;
        }
        $this->description = $description;
        return $this->description;
    }

    public function getDescription() {
        return $this->description;
    }

    protected function validateReturnDescription (MultiLanguageString $description) {
        $description->validate(array(
            'not_empty' => true,
            'max_length' => SponsorReturn::DESCRIPTION_MAX_LENGTH));
    }

    public function getAmount() {
        return $this->amount;
    }

    public function setAmount(MultiCurrencyValue $amount) {
        $this->amount = $amount;
        return $this->amount;
    }

    protected function validateAmount(MultiCurrencyValue $amount) {
        $amount->validate();
    }

        public function getType() {
        return $this->type;
    }

    public function setType(SponsorReturnType $type) {
        $this->type = $type;
        return $this->type;
    }

    public function validate() {
        if (!is_null($this->theme) && !is_a($this->theme, 'Exposure\Model\Theme')) {
            throw new SponsorReturnOnFinancialContributionException(
                    self::EXCEPTION_INVALID_THEME);
        }
        if (!is_a($this->amount, 'Sociable\Model\MultiCurrencyValue')) {
            throw new SponsorReturnOnFinancialContributionException(
                    self::EXCEPTION_INVALID_AMOUNT);
        }
        $this->validateAmount($this->amount);
        if (!is_a($this->description, 'Sociable\Model\MultiLanguageString')) {
            throw new SponsorReturnOnFinancialContributionException(
                    self::EXCEPTION_INVALID_DESCRIPTION);
        }
        $this->validateReturnDescription($this->description);
        if (!is_a($this->type, 'Exposure\Model\SponsorReturnType')) {
            throw new SponsorReturnOnFinancialContributionException(
                    self::EXCEPTION_INVALID_TYPE);
        }
    }
}


text:
import React, {Component, PropTypes } from 'react';
import {
    Form,
    Button,
    Modal,
    Input
} from 'antd';
const FormItem = Form.Item;

class addRegionForm extends Component {
    constructor(props){
      super(props);
    }
    render() {
        let {visible, onCancel, onCreate, region,title } = this.props;
        let {getFieldDecorator} = this.props.form;

        return (
            <Modal
                visible={visible}
                title={title}
                onCancel={onCancel}
                onOk={onCreate}
                footer={[
                    <Button key = "back" size = "large" onClick = {onCancel} > Cancelar </Button>,
                    <Button key="submit" type="primary" size="large" onClick={onCreate}> Aceptar </Button >
                    ]}>
                <Form>
                    {getFieldDecorator('id', {
                        initialValue: region.id?region.id:0
                    })(
                        <Input type="hidden" />
                    )}
                    <FormItem label="Descripción">
                    {getFieldDecorator('name', {
                        rules: [
                            { required: true, message: '¡Por favor ingrese la descripción!' },
                            ],
                            initialValue: region.name?region.name:""
                    })(
                        <Input  placeholder="Descripción de la región" />
                    )}
                    </FormItem>
                </Form>
            </Modal>
        );
    }
}

addRegionForm.propTypes = {
    region: PropTypes.object,
    isSaving: PropTypes.bool.isRequired,
    title: PropTypes.string.isRequired,
    visible: PropTypes.bool,
    onCancel: PropTypes.func.isRequired,
    onCreate: PropTypes.func.isRequired,
};

const addRegion = Form.create()(addRegionForm);

export default addRegion;


text:
import { combineReducers } from 'redux';

const getReducers = requireContext => (
  requireContext.keys().reduce((acc, value, key) => {
    if (key === 0) return acc;

    const domain = value.match(/^\.\/(.+?)\/.+/)[1];

    return {
      ...acc,
      [domain]: requireContext(value).default,
    };
  }, {})
);

export default combineReducers(getReducers(require.context('.', true, /.*^((?!\.test).)*\.js$/)));


text:
package mw.fstraverse.tool;

import java.io.File;

public interface FSProcessor {
    public FProcResult process(File file);
}